2024-11-09 17:25:33,046 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-11-09 17:25:33,057 main DEBUG Took 0.009661 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 17:25:33,058 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 17:25:33,058 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 17:25:33,059 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 17:25:33,060 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,068 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 17:25:33,079 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,081 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,082 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,083 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,084 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,084 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,085 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,086 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,087 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,087 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,088 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,089 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,090 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,090 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,091 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,091 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,092 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,092 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,093 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,093 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,094 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,095 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,095 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 17:25:33,096 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,097 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 17:25:33,099 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 17:25:33,100 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 17:25:33,103 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 17:25:33,104 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 17:25:33,105 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 17:25:33,106 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 17:25:33,115 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 17:25:33,117 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 17:25:33,119 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 17:25:33,119 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 17:25:33,119 main DEBUG createAppenders(={Console}) 2024-11-09 17:25:33,120 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 initialized 2024-11-09 17:25:33,120 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 2024-11-09 17:25:33,121 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7fb4f2a9 OK. 2024-11-09 17:25:33,121 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 17:25:33,122 main DEBUG OutputStream closed 2024-11-09 17:25:33,122 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 17:25:33,122 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 17:25:33,122 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@54e1c68b OK 2024-11-09 17:25:33,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 17:25:33,215 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 17:25:33,217 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 17:25:33,218 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 17:25:33,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 17:25:33,219 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 17:25:33,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 17:25:33,220 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 17:25:33,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 17:25:33,221 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 17:25:33,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 17:25:33,222 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 17:25:33,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 17:25:33,223 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 17:25:33,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 17:25:33,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 17:25:33,224 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 17:25:33,225 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 17:25:33,227 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 17:25:33,227 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@7dda48d9) with optional ClassLoader: null 2024-11-09 17:25:33,228 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 17:25:33,229 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@7dda48d9] started OK. 2024-11-09T17:25:33,467 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156 2024-11-09 17:25:33,470 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 17:25:33,470 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T17:25:33,478 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithBasicPolicy timeout: 13 mins 2024-11-09T17:25:33,503 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T17:25:33,506 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e, deleteOnExit=true 2024-11-09T17:25:33,507 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-09T17:25:33,507 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/test.cache.data in system properties and HBase conf 2024-11-09T17:25:33,508 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T17:25:33,509 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.log.dir in system properties and HBase conf 2024-11-09T17:25:33,509 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T17:25:33,510 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T17:25:33,510 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-09T17:25:33,595 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T17:25:33,678 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T17:25:33,682 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T17:25:33,683 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T17:25:33,683 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T17:25:33,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T17:25:33,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T17:25:33,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T17:25:33,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T17:25:33,685 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T17:25:33,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T17:25:33,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/nfs.dump.dir in system properties and HBase conf 2024-11-09T17:25:33,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/java.io.tmpdir in system properties and HBase conf 2024-11-09T17:25:33,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T17:25:33,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T17:25:33,688 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T17:25:34,612 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T17:25:34,685 INFO [Time-limited test {}] log.Log(170): Logging initialized @2292ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T17:25:34,751 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T17:25:34,821 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T17:25:34,842 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T17:25:34,843 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T17:25:34,844 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T17:25:34,860 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T17:25:34,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.log.dir/,AVAILABLE} 2024-11-09T17:25:34,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T17:25:35,047 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@b03fcff{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/java.io.tmpdir/jetty-localhost-46079-hadoop-hdfs-3_4_1-tests_jar-_-any-14227438794065596657/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T17:25:35,056 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:46079} 2024-11-09T17:25:35,056 INFO [Time-limited test {}] server.Server(415): Started @2664ms 2024-11-09T17:25:35,498 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T17:25:35,506 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T17:25:35,507 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T17:25:35,507 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T17:25:35,507 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T17:25:35,508 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47db50b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.log.dir/,AVAILABLE} 2024-11-09T17:25:35,509 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4727fac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T17:25:35,621 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1f79ec76{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/java.io.tmpdir/jetty-localhost-33169-hadoop-hdfs-3_4_1-tests_jar-_-any-925669573511817916/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T17:25:35,622 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@576ebda6{HTTP/1.1, (http/1.1)}{localhost:33169} 2024-11-09T17:25:35,622 INFO [Time-limited test {}] server.Server(415): Started @3230ms 2024-11-09T17:25:35,677 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T17:25:36,502 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data2/current/BP-237782476-172.17.0.3-1731173134238/current, will proceed with Du for space computation calculation, 2024-11-09T17:25:36,502 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data1/current/BP-237782476-172.17.0.3-1731173134238/current, will proceed with Du for space computation calculation, 2024-11-09T17:25:36,526 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T17:25:36,567 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36badffa144f97e with lease ID 0xb0eaad847d330801: Processing first storage report for DS-fd068ce6-d1f6-4c15-9af8-f61ca7523312 from datanode DatanodeRegistration(127.0.0.1:35365, datanodeUuid=b69a8dc7-51eb-4008-863f-c33ebf910a7f, infoPort=35997, infoSecurePort=0, ipcPort=37487, storageInfo=lv=-57;cid=testClusterID;nsid=1323926910;c=1731173134238) 2024-11-09T17:25:36,568 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36badffa144f97e with lease ID 0xb0eaad847d330801: from storage DS-fd068ce6-d1f6-4c15-9af8-f61ca7523312 node DatanodeRegistration(127.0.0.1:35365, datanodeUuid=b69a8dc7-51eb-4008-863f-c33ebf910a7f, infoPort=35997, infoSecurePort=0, ipcPort=37487, storageInfo=lv=-57;cid=testClusterID;nsid=1323926910;c=1731173134238), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T17:25:36,569 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x36badffa144f97e with lease ID 0xb0eaad847d330801: Processing first storage report for DS-46539aa5-1498-419e-bbae-a655b0d368a9 from datanode DatanodeRegistration(127.0.0.1:35365, datanodeUuid=b69a8dc7-51eb-4008-863f-c33ebf910a7f, infoPort=35997, infoSecurePort=0, ipcPort=37487, storageInfo=lv=-57;cid=testClusterID;nsid=1323926910;c=1731173134238) 2024-11-09T17:25:36,569 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x36badffa144f97e with lease ID 0xb0eaad847d330801: from storage DS-46539aa5-1498-419e-bbae-a655b0d368a9 node DatanodeRegistration(127.0.0.1:35365, datanodeUuid=b69a8dc7-51eb-4008-863f-c33ebf910a7f, infoPort=35997, infoSecurePort=0, ipcPort=37487, storageInfo=lv=-57;cid=testClusterID;nsid=1323926910;c=1731173134238), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T17:25:36,583 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156 2024-11-09T17:25:36,647 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/zookeeper_0, clientPort=57222, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T17:25:36,656 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=57222 2024-11-09T17:25:36,666 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:36,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:36,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741825_1001 (size=7) 2024-11-09T17:25:37,284 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 with version=8 2024-11-09T17:25:37,284 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/hbase-staging 2024-11-09T17:25:37,392 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T17:25:37,617 INFO [Time-limited test {}] client.ConnectionUtils(129): master/c28fd47d982f:0 server-side Connection retries=45 2024-11-09T17:25:37,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:37,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:37,633 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T17:25:37,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:37,633 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T17:25:37,748 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T17:25:37,796 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T17:25:37,804 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T17:25:37,807 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T17:25:37,829 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 44704 (auto-detected) 2024-11-09T17:25:37,830 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T17:25:37,846 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.3:44083 2024-11-09T17:25:37,853 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:37,855 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:37,865 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:44083 connecting to ZooKeeper ensemble=127.0.0.1:57222 2024-11-09T17:25:37,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:440830x0, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T17:25:37,968 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44083-0x10121eb5b6a0000 connected 2024-11-09T17:25:38,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T17:25:38,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T17:25:38,062 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T17:25:38,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44083 2024-11-09T17:25:38,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44083 2024-11-09T17:25:38,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44083 2024-11-09T17:25:38,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44083 2024-11-09T17:25:38,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44083 2024-11-09T17:25:38,074 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7, hbase.cluster.distributed=false 2024-11-09T17:25:38,130 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/c28fd47d982f:0 server-side Connection retries=45 2024-11-09T17:25:38,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:38,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:38,131 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T17:25:38,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T17:25:38,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T17:25:38,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T17:25:38,135 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T17:25:38,136 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.3:45633 2024-11-09T17:25:38,138 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T17:25:38,142 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T17:25:38,143 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:38,145 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:38,149 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45633 connecting to ZooKeeper ensemble=127.0.0.1:57222 2024-11-09T17:25:38,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:456330x0, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T17:25:38,162 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:456330x0, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T17:25:38,162 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45633-0x10121eb5b6a0001 connected 2024-11-09T17:25:38,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T17:25:38,165 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T17:25:38,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45633 2024-11-09T17:25:38,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45633 2024-11-09T17:25:38,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45633 2024-11-09T17:25:38,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45633 2024-11-09T17:25:38,167 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45633 2024-11-09T17:25:38,169 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/c28fd47d982f,44083,1731173137386 2024-11-09T17:25:38,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T17:25:38,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T17:25:38,185 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c28fd47d982f,44083,1731173137386 2024-11-09T17:25:38,185 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c28fd47d982f:44083 2024-11-09T17:25:38,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T17:25:38,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T17:25:38,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:38,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:38,212 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T17:25:38,213 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c28fd47d982f,44083,1731173137386 from backup master directory 2024-11-09T17:25:38,213 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T17:25:38,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T17:25:38,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c28fd47d982f,44083,1731173137386 2024-11-09T17:25:38,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T17:25:38,225 WARN [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T17:25:38,225 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c28fd47d982f,44083,1731173137386 2024-11-09T17:25:38,227 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T17:25:38,229 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T17:25:38,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741826_1002 (size=42) 2024-11-09T17:25:38,702 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/hbase.id with ID: 470270c0-dfea-40a9-8518-0f73323b2604 2024-11-09T17:25:38,748 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T17:25:38,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:38,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:38,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741827_1003 (size=196) 2024-11-09T17:25:39,253 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:25:39,255 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T17:25:39,272 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:39,277 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T17:25:39,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741828_1004 (size=1189) 2024-11-09T17:25:39,735 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store 2024-11-09T17:25:39,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741829_1005 (size=34) 2024-11-09T17:25:40,161 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T17:25:40,161 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:40,162 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T17:25:40,163 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:25:40,163 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:25:40,163 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T17:25:40,163 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:25:40,163 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:25:40,163 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T17:25:40,166 WARN [master/c28fd47d982f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/.initializing 2024-11-09T17:25:40,166 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/WALs/c28fd47d982f,44083,1731173137386 2024-11-09T17:25:40,173 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T17:25:40,182 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28fd47d982f%2C44083%2C1731173137386, suffix=, logDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/WALs/c28fd47d982f,44083,1731173137386, archiveDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/oldWALs, maxLogs=10 2024-11-09T17:25:40,199 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/WALs/c28fd47d982f,44083,1731173137386/c28fd47d982f%2C44083%2C1731173137386.1731173140186, exclude list is [], retry=0 2024-11-09T17:25:40,214 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35365,DS-fd068ce6-d1f6-4c15-9af8-f61ca7523312,DISK] 2024-11-09T17:25:40,216 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T17:25:40,246 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/WALs/c28fd47d982f,44083,1731173137386/c28fd47d982f%2C44083%2C1731173137386.1731173140186 2024-11-09T17:25:40,247 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35997:35997)] 2024-11-09T17:25:40,247 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:25:40,248 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:40,252 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,253 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,289 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,308 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T17:25:40,311 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:40,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:40,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,317 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T17:25:40,317 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:40,318 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:40,318 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,321 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T17:25:40,321 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:40,322 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:40,323 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,325 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T17:25:40,326 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:40,327 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:40,330 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,332 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,340 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T17:25:40,346 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T17:25:40,351 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:25:40,352 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72373559, jitterRate=0.07845006883144379}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T17:25:40,359 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T17:25:40,360 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T17:25:40,384 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31824cd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:40,410 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-09T17:25:40,419 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T17:25:40,419 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T17:25:40,421 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T17:25:40,422 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T17:25:40,426 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 3 msec 2024-11-09T17:25:40,426 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T17:25:40,452 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T17:25:40,466 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T17:25:40,519 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-09T17:25:40,523 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T17:25:40,525 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T17:25:40,532 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-09T17:25:40,535 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T17:25:40,540 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T17:25:40,549 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-09T17:25:40,551 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T17:25:40,561 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T17:25:40,575 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T17:25:40,582 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T17:25:40,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T17:25:40,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T17:25:40,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,591 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=c28fd47d982f,44083,1731173137386, sessionid=0x10121eb5b6a0000, setting cluster-up flag (Was=false) 2024-11-09T17:25:40,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,644 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T17:25:40,648 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c28fd47d982f,44083,1731173137386 2024-11-09T17:25:40,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:40,699 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T17:25:40,702 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c28fd47d982f,44083,1731173137386 2024-11-09T17:25:40,783 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-09T17:25:40,784 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c28fd47d982f:45633 2024-11-09T17:25:40,786 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1008): ClusterId : 470270c0-dfea-40a9-8518-0f73323b2604 2024-11-09T17:25:40,788 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T17:25:40,788 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-09T17:25:40,791 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T17:25:40,795 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c28fd47d982f,44083,1731173137386 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T17:25:40,798 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c28fd47d982f:0, corePoolSize=5, maxPoolSize=5 2024-11-09T17:25:40,798 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c28fd47d982f:0, corePoolSize=5, maxPoolSize=5 2024-11-09T17:25:40,798 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c28fd47d982f:0, corePoolSize=5, maxPoolSize=5 2024-11-09T17:25:40,798 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c28fd47d982f:0, corePoolSize=5, maxPoolSize=5 2024-11-09T17:25:40,798 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c28fd47d982f:0, corePoolSize=10, maxPoolSize=10 2024-11-09T17:25:40,799 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,799 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c28fd47d982f:0, corePoolSize=2, maxPoolSize=2 2024-11-09T17:25:40,799 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,801 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731173170800 2024-11-09T17:25:40,801 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T17:25:40,801 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T17:25:40,802 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T17:25:40,803 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T17:25:40,804 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T17:25:40,805 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-09T17:25:40,806 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T17:25:40,807 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T17:25:40,807 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T17:25:40,807 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T17:25:40,809 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,809 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:40,809 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T17:25:40,810 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T17:25:40,811 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T17:25:40,812 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T17:25:40,812 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T17:25:40,813 DEBUG [RS:0;c28fd47d982f:45633 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b0ccb61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:40,814 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T17:25:40,814 DEBUG [RS:0;c28fd47d982f:45633 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34df9471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c28fd47d982f/172.17.0.3:0 2024-11-09T17:25:40,814 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T17:25:40,816 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.large.0-1731173140816,5,FailOnTimeoutGroup] 2024-11-09T17:25:40,816 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.small.0-1731173140816,5,FailOnTimeoutGroup] 2024-11-09T17:25:40,817 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,817 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T17:25:40,817 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-09T17:25:40,818 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-09T17:25:40,818 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-09T17:25:40,818 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,818 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,820 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(3073): reportForDuty to master=c28fd47d982f,44083,1731173137386 with isa=c28fd47d982f/172.17.0.3:45633, startcode=1731173138129 2024-11-09T17:25:40,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741831_1007 (size=1039) 2024-11-09T17:25:40,830 DEBUG [RS:0;c28fd47d982f:45633 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T17:25:40,865 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38101, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T17:25:40,871 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44083 {}] master.ServerManager(332): Checking decommissioned status of RegionServer c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,873 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44083 {}] master.ServerManager(486): Registering regionserver=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,885 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:25:40,885 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:46491 2024-11-09T17:25:40,886 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-09T17:25:40,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T17:25:40,895 DEBUG [RS:0;c28fd47d982f:45633 {}] zookeeper.ZKUtil(111): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,895 WARN [RS:0;c28fd47d982f:45633 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T17:25:40,895 INFO [RS:0;c28fd47d982f:45633 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T17:25:40,895 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,897 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c28fd47d982f,45633,1731173138129] 2024-11-09T17:25:40,911 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-09T17:25:40,921 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T17:25:40,935 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T17:25:40,937 INFO [RS:0;c28fd47d982f:45633 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T17:25:40,937 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,938 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-09T17:25:40,943 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,944 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c28fd47d982f:0, corePoolSize=2, maxPoolSize=2 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c28fd47d982f:0, corePoolSize=1, maxPoolSize=1 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c28fd47d982f:0, corePoolSize=3, maxPoolSize=3 2024-11-09T17:25:40,945 DEBUG [RS:0;c28fd47d982f:45633 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0, corePoolSize=3, maxPoolSize=3 2024-11-09T17:25:40,946 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,946 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,946 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,947 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,947 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,45633,1731173138129-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T17:25:40,964 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T17:25:40,966 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,45633,1731173138129-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:40,983 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.Replication(204): c28fd47d982f,45633,1731173138129 started 2024-11-09T17:25:40,983 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1767): Serving as c28fd47d982f,45633,1731173138129, RpcServer on c28fd47d982f/172.17.0.3:45633, sessionid=0x10121eb5b6a0001 2024-11-09T17:25:40,984 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T17:25:40,984 DEBUG [RS:0;c28fd47d982f:45633 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,984 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c28fd47d982f,45633,1731173138129' 2024-11-09T17:25:40,984 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T17:25:40,985 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T17:25:40,985 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T17:25:40,985 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T17:25:40,986 DEBUG [RS:0;c28fd47d982f:45633 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c28fd47d982f,45633,1731173138129 2024-11-09T17:25:40,986 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c28fd47d982f,45633,1731173138129' 2024-11-09T17:25:40,986 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T17:25:40,986 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T17:25:40,987 DEBUG [RS:0;c28fd47d982f:45633 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T17:25:40,987 INFO [RS:0;c28fd47d982f:45633 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T17:25:40,987 INFO [RS:0;c28fd47d982f:45633 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T17:25:41,094 INFO [RS:0;c28fd47d982f:45633 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T17:25:41,098 INFO [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28fd47d982f%2C45633%2C1731173138129, suffix=, logDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129, archiveDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/oldWALs, maxLogs=32 2024-11-09T17:25:41,113 DEBUG [RS:0;c28fd47d982f:45633 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129/c28fd47d982f%2C45633%2C1731173138129.1731173141101, exclude list is [], retry=0 2024-11-09T17:25:41,118 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35365,DS-fd068ce6-d1f6-4c15-9af8-f61ca7523312,DISK] 2024-11-09T17:25:41,121 INFO [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129/c28fd47d982f%2C45633%2C1731173138129.1731173141101 2024-11-09T17:25:41,121 DEBUG [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35997:35997)] 2024-11-09T17:25:41,226 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-09T17:25:41,227 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:25:41,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741833_1009 (size=32) 2024-11-09T17:25:41,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:41,649 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T17:25:41,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T17:25:41,653 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:41,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:41,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T17:25:41,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T17:25:41,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:41,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:41,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T17:25:41,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T17:25:41,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:41,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:41,666 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740 2024-11-09T17:25:41,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740 2024-11-09T17:25:41,671 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:25:41,673 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T17:25:41,677 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:25:41,678 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60276540, jitterRate=-0.101809561252594}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:25:41,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T17:25:41,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T17:25:41,680 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T17:25:41,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T17:25:41,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T17:25:41,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T17:25:41,681 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T17:25:41,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T17:25:41,684 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-09T17:25:41,684 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-09T17:25:41,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T17:25:41,695 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T17:25:41,698 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T17:25:41,852 DEBUG [c28fd47d982f:44083 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T17:25:41,860 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:41,865 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c28fd47d982f,45633,1731173138129, state=OPENING 2024-11-09T17:25:41,911 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T17:25:41,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:41,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:41,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T17:25:41,921 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T17:25:41,926 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:25:42,108 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:42,109 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T17:25:42,112 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46640, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T17:25:42,121 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-09T17:25:42,121 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T17:25:42,122 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T17:25:42,125 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c28fd47d982f%2C45633%2C1731173138129.meta, suffix=.meta, logDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129, archiveDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/oldWALs, maxLogs=32 2024-11-09T17:25:42,141 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129/c28fd47d982f%2C45633%2C1731173138129.meta.1731173142127.meta, exclude list is [], retry=0 2024-11-09T17:25:42,145 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35365,DS-fd068ce6-d1f6-4c15-9af8-f61ca7523312,DISK] 2024-11-09T17:25:42,147 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/WALs/c28fd47d982f,45633,1731173138129/c28fd47d982f%2C45633%2C1731173138129.meta.1731173142127.meta 2024-11-09T17:25:42,148 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35997:35997)] 2024-11-09T17:25:42,148 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:25:42,150 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T17:25:42,201 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T17:25:42,205 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T17:25:42,208 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T17:25:42,208 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:42,208 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-09T17:25:42,209 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-09T17:25:42,212 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T17:25:42,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T17:25:42,213 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:42,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:42,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T17:25:42,216 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T17:25:42,216 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:42,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:42,217 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T17:25:42,218 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T17:25:42,218 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:42,219 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T17:25:42,221 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740 2024-11-09T17:25:42,224 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740 2024-11-09T17:25:42,226 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:25:42,229 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-09T17:25:42,231 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70080528, jitterRate=0.0442812442779541}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:25:42,233 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-09T17:25:42,240 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731173142104 2024-11-09T17:25:42,250 DEBUG [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T17:25:42,250 INFO [RS_OPEN_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-09T17:25:42,251 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:42,253 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c28fd47d982f,45633,1731173138129, state=OPEN 2024-11-09T17:25:42,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T17:25:42,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T17:25:42,314 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T17:25:42,314 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T17:25:42,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T17:25:42,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=c28fd47d982f,45633,1731173138129 in 387 msec 2024-11-09T17:25:42,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T17:25:42,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 635 msec 2024-11-09T17:25:42,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.5930 sec 2024-11-09T17:25:42,333 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731173142333, completionTime=-1 2024-11-09T17:25:42,334 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T17:25:42,334 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-09T17:25:42,367 DEBUG [hconnection-0x4a2c6199-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:42,369 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46652, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:42,379 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-09T17:25:42,379 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731173202379 2024-11-09T17:25:42,379 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731173262379 2024-11-09T17:25:42,380 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 45 msec 2024-11-09T17:25:42,423 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:42,423 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:42,423 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:42,424 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c28fd47d982f:44083, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:42,425 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:42,429 DEBUG [master/c28fd47d982f:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-09T17:25:42,433 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-09T17:25:42,434 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T17:25:42,440 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-09T17:25:42,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:25:42,443 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:42,445 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:25:42,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741835_1011 (size=358) 2024-11-09T17:25:42,867 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 8909151a42b30d353536c39084e3c0b3, NAME => 'hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:25:42,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741836_1012 (size=42) 2024-11-09T17:25:43,280 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:43,281 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 8909151a42b30d353536c39084e3c0b3, disabling compactions & flushes 2024-11-09T17:25:43,281 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,282 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,282 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. after waiting 0 ms 2024-11-09T17:25:43,282 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,282 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,283 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 8909151a42b30d353536c39084e3c0b3: 2024-11-09T17:25:43,288 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:25:43,314 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731173143309"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173143309"}]},"ts":"1731173143309"} 2024-11-09T17:25:43,342 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:25:43,344 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:25:43,346 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173143344"}]},"ts":"1731173143344"} 2024-11-09T17:25:43,350 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-09T17:25:43,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8909151a42b30d353536c39084e3c0b3, ASSIGN}] 2024-11-09T17:25:43,408 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=8909151a42b30d353536c39084e3c0b3, ASSIGN 2024-11-09T17:25:43,410 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=8909151a42b30d353536c39084e3c0b3, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:25:43,561 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8909151a42b30d353536c39084e3c0b3, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:43,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 8909151a42b30d353536c39084e3c0b3, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:25:43,723 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:43,733 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,733 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 8909151a42b30d353536c39084e3c0b3, NAME => 'hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:25:43,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:43,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,734 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,736 INFO [StoreOpener-8909151a42b30d353536c39084e3c0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,739 INFO [StoreOpener-8909151a42b30d353536c39084e3c0b3-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8909151a42b30d353536c39084e3c0b3 columnFamilyName info 2024-11-09T17:25:43,739 DEBUG [StoreOpener-8909151a42b30d353536c39084e3c0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:43,740 INFO [StoreOpener-8909151a42b30d353536c39084e3c0b3-1 {}] regionserver.HStore(327): Store=8909151a42b30d353536c39084e3c0b3/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:43,741 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,742 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,745 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:25:43,749 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:25:43,750 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 8909151a42b30d353536c39084e3c0b3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66397093, jitterRate=-0.01060621440410614}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T17:25:43,751 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 8909151a42b30d353536c39084e3c0b3: 2024-11-09T17:25:43,753 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3., pid=6, masterSystemTime=1731173143723 2024-11-09T17:25:43,756 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,756 INFO [RS_OPEN_PRIORITY_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:25:43,757 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=8909151a42b30d353536c39084e3c0b3, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:43,759 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44083 {}] assignment.AssignmentManager(1526): Unable to acquire lock for regionNode state=OPEN, location=c28fd47d982f,45633,1731173138129, table=hbase:namespace, region=8909151a42b30d353536c39084e3c0b3. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T17:25:43,765 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T17:25:43,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 8909151a42b30d353536c39084e3c0b3, server=c28fd47d982f,45633,1731173138129 in 195 msec 2024-11-09T17:25:43,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T17:25:43,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=8909151a42b30d353536c39084e3c0b3, ASSIGN in 361 msec 2024-11-09T17:25:43,770 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:25:43,770 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173143770"}]},"ts":"1731173143770"} 2024-11-09T17:25:43,772 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-09T17:25:43,813 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:25:43,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3780 sec 2024-11-09T17:25:43,845 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-09T17:25:43,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-09T17:25:43,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:43,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:25:43,882 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-09T17:25:43,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T17:25:43,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 35 msec 2024-11-09T17:25:43,928 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-09T17:25:43,949 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-09T17:25:43,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 32 msec 2024-11-09T17:25:43,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-09T17:25:44,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-09T17:25:44,008 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.782sec 2024-11-09T17:25:44,010 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T17:25:44,012 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T17:25:44,014 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T17:25:44,015 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T17:25:44,015 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T17:25:44,016 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T17:25:44,016 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T17:25:44,022 DEBUG [master/c28fd47d982f:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-09T17:25:44,023 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T17:25:44,023 INFO [master/c28fd47d982f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c28fd47d982f,44083,1731173137386-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T17:25:44,091 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e541e88 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e83c466 2024-11-09T17:25:44,092 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-09T17:25:44,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305a704f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:44,107 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T17:25:44,107 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T17:25:44,116 DEBUG [hconnection-0x68773b0e-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:44,124 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46656, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:44,132 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=c28fd47d982f,44083,1731173137386 2024-11-09T17:25:44,145 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMixedAtomicity Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=174, ProcessCount=11, AvailableMemoryMB=6592 2024-11-09T17:25:44,153 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:25:44,155 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55988, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:25:44,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:25:44,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:25:44,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:25:44,168 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:25:44,168 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:44,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 9 2024-11-09T17:25:44,170 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:25:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:44,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741837_1013 (size=960) 2024-11-09T17:25:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:44,587 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:25:44,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741838_1014 (size=53) 2024-11-09T17:25:44,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:45,002 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:45,003 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing d0b59f7cee8e92155959d9919114a0a2, disabling compactions & flushes 2024-11-09T17:25:45,003 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,003 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,003 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. after waiting 0 ms 2024-11-09T17:25:45,003 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:45,008 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:25:45,009 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173145009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173145009"}]},"ts":"1731173145009"} 2024-11-09T17:25:45,015 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:25:45,017 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:25:45,018 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173145017"}]},"ts":"1731173145017"} 2024-11-09T17:25:45,021 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:25:45,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, ASSIGN}] 2024-11-09T17:25:45,075 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, ASSIGN 2024-11-09T17:25:45,078 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=10, ppid=9, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:25:45,229 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d0b59f7cee8e92155959d9919114a0a2, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:45,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE; OpenRegionProcedure d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:25:45,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:45,393 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:45,406 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,406 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7285): Opening region: {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:25:45,407 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,407 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:25:45,407 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7327): checking encryption for d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,407 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7330): checking classloading for d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,409 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,412 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:25:45,413 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0b59f7cee8e92155959d9919114a0a2 columnFamilyName A 2024-11-09T17:25:45,413 DEBUG [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:45,414 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(327): Store=d0b59f7cee8e92155959d9919114a0a2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:45,414 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,417 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:25:45,417 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0b59f7cee8e92155959d9919114a0a2 columnFamilyName B 2024-11-09T17:25:45,417 DEBUG [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:45,418 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(327): Store=d0b59f7cee8e92155959d9919114a0a2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:45,419 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,421 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:25:45,421 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0b59f7cee8e92155959d9919114a0a2 columnFamilyName C 2024-11-09T17:25:45,421 DEBUG [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:25:45,422 INFO [StoreOpener-d0b59f7cee8e92155959d9919114a0a2-1 {}] regionserver.HStore(327): Store=d0b59f7cee8e92155959d9919114a0a2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:25:45,423 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,424 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,425 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,427 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:25:45,430 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1085): writing seq id for d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:45,434 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:25:45,435 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1102): Opened d0b59f7cee8e92155959d9919114a0a2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68001543, jitterRate=0.013301953673362732}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:25:45,435 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1001): Region open journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:45,437 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., pid=11, masterSystemTime=1731173145393 2024-11-09T17:25:45,439 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,440 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:45,440 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=10 updating hbase:meta row=d0b59f7cee8e92155959d9919114a0a2, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:45,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=10 2024-11-09T17:25:45,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=10, state=SUCCESS; OpenRegionProcedure d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 in 207 msec 2024-11-09T17:25:45,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T17:25:45,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, ASSIGN in 376 msec 2024-11-09T17:25:45,450 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:25:45,450 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173145450"}]},"ts":"1731173145450"} 2024-11-09T17:25:45,452 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:25:45,462 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=9, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:25:45,465 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2980 sec 2024-11-09T17:25:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-09T17:25:46,301 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 9 completed 2024-11-09T17:25:46,315 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f6e36fe to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e98ea32 2024-11-09T17:25:46,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b9fcedf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,371 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46672, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:25:46,379 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56002, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:25:46,387 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f343a4d to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12885408 2024-11-09T17:25:46,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9bd0964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,401 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22cb07dd to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72b32f98 2024-11-09T17:25:46,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18cb251d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,415 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x478bae6b to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4977266 2024-11-09T17:25:46,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45b55c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,428 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5400112e to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bbb5d8a 2024-11-09T17:25:46,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e52b42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,442 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38766d64 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@18603bb9 2024-11-09T17:25:46,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3883f7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,455 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x295cb1ac to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72e97e4b 2024-11-09T17:25:46,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a1285d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,467 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70267494 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@490457fd 2024-11-09T17:25:46,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@527c6d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,480 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d2a8e08 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c8de680 2024-11-09T17:25:46,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47fe2fa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,492 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c915d17 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f6b07e3 2024-11-09T17:25:46,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595e9ebe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:25:46,508 DEBUG [hconnection-0x593641ee-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,508 DEBUG [hconnection-0x4766e894-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,509 DEBUG [hconnection-0x505cac31-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,510 DEBUG [hconnection-0x4d0fdb91-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,511 DEBUG [hconnection-0x74f48c42-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,511 DEBUG [hconnection-0x1edc7dcf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,513 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,513 DEBUG [hconnection-0x2e14f92e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,514 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46704, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,514 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46688, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,514 DEBUG [hconnection-0xabff582-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:46,516 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46714, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,518 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46722, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,518 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46738, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,520 DEBUG [hconnection-0x4c038b40-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:25:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees 2024-11-09T17:25:46,522 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46744, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,524 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:46,525 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46758, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:46,527 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=12, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:46,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=12, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:46,534 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46768, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:25:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:46,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:25:46,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:46,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:46,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:46,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:46,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:46,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:46,695 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:46,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173206696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173206693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173206696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173206701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173206701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:46,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:46,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4385ce6a9f334720a51d1642fe63f76d is 50, key is test_row_0/A:col10/1731173146575/Put/seqid=0 2024-11-09T17:25:46,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741839_1015 (size=14341) 2024-11-09T17:25:46,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173206846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173206847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173206847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173206847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173206848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,888 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:46,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:46,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:46,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:46,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:46,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:46,917 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T17:25:46,919 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T17:25:46,920 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-09T17:25:47,054 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173207053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173207054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173207053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173207054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173207055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:47,218 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4385ce6a9f334720a51d1642fe63f76d 2024-11-09T17:25:47,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/db016d2a30154160a45a958111c2ce04 is 50, key is test_row_0/B:col10/1731173146575/Put/seqid=0 2024-11-09T17:25:47,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741840_1016 (size=12001) 2024-11-09T17:25:47,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/db016d2a30154160a45a958111c2ce04 2024-11-09T17:25:47,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173207360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173207360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173207360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173207361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173207363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,377 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/41a9bfa19ba94713a3ad25a78c199d23 is 50, key is test_row_0/C:col10/1731173146575/Put/seqid=0 2024-11-09T17:25:47,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741841_1017 (size=12001) 2024-11-09T17:25:47,532 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:47,686 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,721 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:25:47,806 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T17:25:47,806 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T17:25:47,809 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-09T17:25:47,809 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-09T17:25:47,810 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T17:25:47,811 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T17:25:47,812 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T17:25:47,813 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T17:25:47,814 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T17:25:47,814 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T17:25:47,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/41a9bfa19ba94713a3ad25a78c199d23 2024-11-09T17:25:47,842 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:47,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:47,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] handler.RSProcedureHandler(58): pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=13 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=13 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:47,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4385ce6a9f334720a51d1642fe63f76d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d 2024-11-09T17:25:47,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173207867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173207867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173207868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173207869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:47,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173207869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d, entries=200, sequenceid=17, filesize=14.0 K 2024-11-09T17:25:47,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/db016d2a30154160a45a958111c2ce04 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04 2024-11-09T17:25:47,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:25:47,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/41a9bfa19ba94713a3ad25a78c199d23 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23 2024-11-09T17:25:47,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:25:47,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d0b59f7cee8e92155959d9919114a0a2 in 1328ms, sequenceid=17, compaction requested=false 2024-11-09T17:25:47,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:47,998 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:47,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=13 2024-11-09T17:25:48,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:48,000 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:25:48,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/276b1c4ad1584194b19a36cf59c3ebb2 is 50, key is test_row_0/A:col10/1731173146693/Put/seqid=0 2024-11-09T17:25:48,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741842_1018 (size=12001) 2024-11-09T17:25:48,043 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/276b1c4ad1584194b19a36cf59c3ebb2 2024-11-09T17:25:48,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/cb28b15ff01c423ba469ffd6eff365ce is 50, key is test_row_0/B:col10/1731173146693/Put/seqid=0 2024-11-09T17:25:48,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741843_1019 (size=12001) 2024-11-09T17:25:48,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/cb28b15ff01c423ba469ffd6eff365ce 2024-11-09T17:25:48,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/01113e86195e4f1fbaa3b31ffa6ad223 is 50, key is test_row_0/C:col10/1731173146693/Put/seqid=0 2024-11-09T17:25:48,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741844_1020 (size=12001) 2024-11-09T17:25:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/01113e86195e4f1fbaa3b31ffa6ad223 2024-11-09T17:25:48,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/276b1c4ad1584194b19a36cf59c3ebb2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2 2024-11-09T17:25:48,191 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T17:25:48,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/cb28b15ff01c423ba469ffd6eff365ce as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce 2024-11-09T17:25:48,213 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T17:25:48,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/01113e86195e4f1fbaa3b31ffa6ad223 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223 2024-11-09T17:25:48,245 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223, entries=150, sequenceid=37, filesize=11.7 K 2024-11-09T17:25:48,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for d0b59f7cee8e92155959d9919114a0a2 in 246ms, sequenceid=37, compaction requested=false 2024-11-09T17:25:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:48,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=13}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=13 2024-11-09T17:25:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=13 2024-11-09T17:25:48,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=12 2024-11-09T17:25:48,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7200 sec 2024-11-09T17:25:48,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=12, table=TestAcidGuarantees in 1.7380 sec 2024-11-09T17:25:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=12 2024-11-09T17:25:48,637 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 12 completed 2024-11-09T17:25:48,640 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees 2024-11-09T17:25:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-09T17:25:48,643 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:48,645 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=14, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:48,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-09T17:25:48,799 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=15 2024-11-09T17:25:48,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:48,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-09T17:25:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=15 2024-11-09T17:25:48,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-11-09T17:25:48,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 157 msec 2024-11-09T17:25:48,812 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=14, table=TestAcidGuarantees in 170 msec 2024-11-09T17:25:48,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:48,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:48,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:48,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/66458c15bd1d48b994068f175bcbff27 is 50, key is test_row_0/A:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:48,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741845_1021 (size=16681) 2024-11-09T17:25:48,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/66458c15bd1d48b994068f175bcbff27 2024-11-09T17:25:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=14 2024-11-09T17:25:48,946 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 14 completed 2024-11-09T17:25:48,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees 2024-11-09T17:25:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:48,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:48,958 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=16, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:48,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:48,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173208958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:48,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173208962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:48,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173208963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:48,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/d111349409e2484a9ad2dcf76a2beef2 is 50, key is test_row_0/B:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173208967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173208969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:48,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741846_1022 (size=12001) 2024-11-09T17:25:49,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/d111349409e2484a9ad2dcf76a2beef2 2024-11-09T17:25:49,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c4f55121e9f04a45a4267aefb11afdb0 is 50, key is test_row_0/C:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:49,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741847_1023 (size=12001) 2024-11-09T17:25:49,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c4f55121e9f04a45a4267aefb11afdb0 2024-11-09T17:25:49,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:49,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/66458c15bd1d48b994068f175bcbff27 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27 2024-11-09T17:25:49,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173209072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173209074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173209076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173209077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27, entries=250, sequenceid=49, filesize=16.3 K 2024-11-09T17:25:49,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173209079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/d111349409e2484a9ad2dcf76a2beef2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2 2024-11-09T17:25:49,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T17:25:49,112 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-09T17:25:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:49,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] handler.RSProcedureHandler(58): pid=17 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:49,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=17 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:49,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=17 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:49,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c4f55121e9f04a45a4267aefb11afdb0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0 2024-11-09T17:25:49,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T17:25:49,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for d0b59f7cee8e92155959d9919114a0a2 in 252ms, sequenceid=49, compaction requested=true 2024-11-09T17:25:49,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:49,151 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:49,151 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:49,155 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:49,157 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:49,157 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,157 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.2 K 2024-11-09T17:25:49,159 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting db016d2a30154160a45a958111c2ce04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173146575 2024-11-09T17:25:49,161 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting cb28b15ff01c423ba469ffd6eff365ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731173146691 2024-11-09T17:25:49,161 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:49,161 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:49,161 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,161 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d111349409e2484a9ad2dcf76a2beef2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148886 2024-11-09T17:25:49,162 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=42.0 K 2024-11-09T17:25:49,163 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4385ce6a9f334720a51d1642fe63f76d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173146575 2024-11-09T17:25:49,166 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 276b1c4ad1584194b19a36cf59c3ebb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731173146691 2024-11-09T17:25:49,167 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 66458c15bd1d48b994068f175bcbff27, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148885 2024-11-09T17:25:49,211 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:49,212 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3c48ea2dfa724a2d8139aea597fb723d is 50, key is test_row_0/A:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:49,214 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:49,215 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/b1d7b7a3ecb646fc9eae23079fe68b9c is 50, key is test_row_0/B:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:49,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741848_1024 (size=12104) 2024-11-09T17:25:49,250 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/b1d7b7a3ecb646fc9eae23079fe68b9c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b1d7b7a3ecb646fc9eae23079fe68b9c 2024-11-09T17:25:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:49,266 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=17 2024-11-09T17:25:49,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,268 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T17:25:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:49,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741849_1025 (size=12104) 2024-11-09T17:25:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:49,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:49,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:49,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:49,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:49,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/37ccf8bb3a9e4d4fb25469bc7643fb0b is 50, key is test_row_0/A:col10/1731173148962/Put/seqid=0 2024-11-09T17:25:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:49,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:49,285 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into b1d7b7a3ecb646fc9eae23079fe68b9c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:49,285 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:49,285 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173149151; duration=0sec 2024-11-09T17:25:49,286 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:49,288 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:49,288 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:49,291 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:49,291 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:49,292 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:49,292 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.2 K 2024-11-09T17:25:49,294 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 41a9bfa19ba94713a3ad25a78c199d23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173146575 2024-11-09T17:25:49,295 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 01113e86195e4f1fbaa3b31ffa6ad223, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731173146691 2024-11-09T17:25:49,296 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c4f55121e9f04a45a4267aefb11afdb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148886 2024-11-09T17:25:49,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741850_1026 (size=12001) 2024-11-09T17:25:49,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173209306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173209307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173209309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173209309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,324 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:49,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173209318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,324 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/bbb7b79b693f453b8d566caccb17c917 is 50, key is test_row_0/C:col10/1731173148893/Put/seqid=0 2024-11-09T17:25:49,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741851_1027 (size=12104) 2024-11-09T17:25:49,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173209422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173209423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173209423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173209424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173209432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:49,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173209633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173209633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173209634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173209637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173209639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,685 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3c48ea2dfa724a2d8139aea597fb723d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3c48ea2dfa724a2d8139aea597fb723d 2024-11-09T17:25:49,706 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 3c48ea2dfa724a2d8139aea597fb723d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:49,706 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:49,706 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173149149; duration=0sec 2024-11-09T17:25:49,707 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:49,707 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:49,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/37ccf8bb3a9e4d4fb25469bc7643fb0b 2024-11-09T17:25:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4ceed7bd6a6c4d3db08237423b355523 is 50, key is test_row_0/B:col10/1731173148962/Put/seqid=0 2024-11-09T17:25:49,759 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/bbb7b79b693f453b8d566caccb17c917 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/bbb7b79b693f453b8d566caccb17c917 2024-11-09T17:25:49,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741852_1028 (size=12001) 2024-11-09T17:25:49,777 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into bbb7b79b693f453b8d566caccb17c917(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:49,778 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:49,778 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173149151; duration=0sec 2024-11-09T17:25:49,778 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:49,778 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:49,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173209938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173209940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173209941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173209941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:49,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173209944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:50,174 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4ceed7bd6a6c4d3db08237423b355523 2024-11-09T17:25:50,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/d758138668ea4fe1a1eae78087b76e56 is 50, key is test_row_0/C:col10/1731173148962/Put/seqid=0 2024-11-09T17:25:50,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741853_1029 (size=12001) 2024-11-09T17:25:50,217 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/d758138668ea4fe1a1eae78087b76e56 2024-11-09T17:25:50,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/37ccf8bb3a9e4d4fb25469bc7643fb0b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b 2024-11-09T17:25:50,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T17:25:50,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4ceed7bd6a6c4d3db08237423b355523 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523 2024-11-09T17:25:50,278 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T17:25:50,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/d758138668ea4fe1a1eae78087b76e56 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56 2024-11-09T17:25:50,301 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56, entries=150, sequenceid=75, filesize=11.7 K 2024-11-09T17:25:50,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d0b59f7cee8e92155959d9919114a0a2 in 1036ms, sequenceid=75, compaction requested=false 2024-11-09T17:25:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:50,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:50,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=17}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=17 2024-11-09T17:25:50,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=17 2024-11-09T17:25:50,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=17, resume processing ppid=16 2024-11-09T17:25:50,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, ppid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3470 sec 2024-11-09T17:25:50,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=16, table=TestAcidGuarantees in 1.3640 sec 2024-11-09T17:25:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:50,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:50,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/54c03629fa6d467a995b424c9ca8de53 is 50, key is test_row_0/A:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741854_1030 (size=12001) 2024-11-09T17:25:50,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/54c03629fa6d467a995b424c9ca8de53 2024-11-09T17:25:50,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a7830f241d8b42b0b850afbd926d8de4 is 50, key is test_row_0/B:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173210506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173210508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173210510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173210514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173210515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741855_1031 (size=12001) 2024-11-09T17:25:50,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a7830f241d8b42b0b850afbd926d8de4 2024-11-09T17:25:50,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ca878331f14b474c904019549a1d0915 is 50, key is test_row_0/C:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741856_1032 (size=12001) 2024-11-09T17:25:50,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ca878331f14b474c904019549a1d0915 2024-11-09T17:25:50,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/54c03629fa6d467a995b424c9ca8de53 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53 2024-11-09T17:25:50,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T17:25:50,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a7830f241d8b42b0b850afbd926d8de4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4 2024-11-09T17:25:50,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T17:25:50,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ca878331f14b474c904019549a1d0915 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915 2024-11-09T17:25:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173210618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173210618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173210618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173210623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173210623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915, entries=150, sequenceid=89, filesize=11.7 K 2024-11-09T17:25:50,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d0b59f7cee8e92155959d9919114a0a2 in 190ms, sequenceid=89, compaction requested=true 2024-11-09T17:25:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:50,637 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:50,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:50,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:50,638 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:50,641 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:50,641 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:50,641 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:50,641 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:50,641 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3c48ea2dfa724a2d8139aea597fb723d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.3 K 2024-11-09T17:25:50,641 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b1d7b7a3ecb646fc9eae23079fe68b9c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.3 K 2024-11-09T17:25:50,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:50,642 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting b1d7b7a3ecb646fc9eae23079fe68b9c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148886 2024-11-09T17:25:50,642 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3c48ea2dfa724a2d8139aea597fb723d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148886 2024-11-09T17:25:50,643 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 37ccf8bb3a9e4d4fb25469bc7643fb0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731173148949 2024-11-09T17:25:50,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:50,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:50,644 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4ceed7bd6a6c4d3db08237423b355523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731173148949 2024-11-09T17:25:50,645 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 54c03629fa6d467a995b424c9ca8de53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:50,645 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a7830f241d8b42b0b850afbd926d8de4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:50,666 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:50,667 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/65a10973659248c28c5ae99074a6907b is 50, key is test_row_0/B:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,681 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:50,682 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3927dcd5e6584f93bc5642c7e53758a9 is 50, key is test_row_0/A:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741857_1033 (size=12207) 2024-11-09T17:25:50,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741858_1034 (size=12207) 2024-11-09T17:25:50,711 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3927dcd5e6584f93bc5642c7e53758a9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3927dcd5e6584f93bc5642c7e53758a9 2024-11-09T17:25:50,723 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 3927dcd5e6584f93bc5642c7e53758a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:50,723 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:50,723 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173150637; duration=0sec 2024-11-09T17:25:50,723 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:50,723 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:50,723 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:50,726 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:50,727 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:50,727 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:50,727 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/bbb7b79b693f453b8d566caccb17c917, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.3 K 2024-11-09T17:25:50,728 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting bbb7b79b693f453b8d566caccb17c917, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731173148886 2024-11-09T17:25:50,729 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d758138668ea4fe1a1eae78087b76e56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731173148949 2024-11-09T17:25:50,730 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting ca878331f14b474c904019549a1d0915, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:50,759 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:50,760 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/77003a63983044d3ae706268ac6c39d9 is 50, key is test_row_0/C:col10/1731173149306/Put/seqid=0 2024-11-09T17:25:50,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741859_1035 (size=12207) 2024-11-09T17:25:50,800 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/77003a63983044d3ae706268ac6c39d9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/77003a63983044d3ae706268ac6c39d9 2024-11-09T17:25:50,814 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 77003a63983044d3ae706268ac6c39d9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:50,814 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:50,814 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173150643; duration=0sec 2024-11-09T17:25:50,815 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:50,815 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:50,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:50,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:50,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1855c3a3f68a4137bff61cb4fd4427c1 is 50, key is test_row_0/A:col10/1731173150507/Put/seqid=0 2024-11-09T17:25:50,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173210835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173210835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173210841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173210843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173210843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741860_1036 (size=12001) 2024-11-09T17:25:50,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1855c3a3f68a4137bff61cb4fd4427c1 2024-11-09T17:25:50,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/b74da3fd5888449586053afebece3093 is 50, key is test_row_0/B:col10/1731173150507/Put/seqid=0 2024-11-09T17:25:50,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741861_1037 (size=12001) 2024-11-09T17:25:50,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/b74da3fd5888449586053afebece3093 2024-11-09T17:25:50,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fbe7dc89f5db457ca8485cd16ddcafb3 is 50, key is test_row_0/C:col10/1731173150507/Put/seqid=0 2024-11-09T17:25:50,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741862_1038 (size=12001) 2024-11-09T17:25:50,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173210949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173210951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173210952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:50,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173210952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=16 2024-11-09T17:25:51,063 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 16 completed 2024-11-09T17:25:51,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees 2024-11-09T17:25:51,072 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:51,073 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=18, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:51,074 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:51,093 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/65a10973659248c28c5ae99074a6907b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/65a10973659248c28c5ae99074a6907b 2024-11-09T17:25:51,105 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 65a10973659248c28c5ae99074a6907b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:51,105 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:51,105 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173150638; duration=0sec 2024-11-09T17:25:51,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:51,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:51,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173211147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173211153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173211155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173211155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173211162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:51,226 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:51,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,338 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fbe7dc89f5db457ca8485cd16ddcafb3 2024-11-09T17:25:51,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1855c3a3f68a4137bff61cb4fd4427c1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1 2024-11-09T17:25:51,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T17:25:51,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/b74da3fd5888449586053afebece3093 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093 2024-11-09T17:25:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:51,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T17:25:51,381 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fbe7dc89f5db457ca8485cd16ddcafb3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3 2024-11-09T17:25:51,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3, entries=150, sequenceid=118, filesize=11.7 K 2024-11-09T17:25:51,400 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for d0b59f7cee8e92155959d9919114a0a2 in 574ms, sequenceid=118, compaction requested=false 2024-11-09T17:25:51,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:51,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:51,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:51,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e8c5c32863d4f7b842548fbed2d83b6 is 50, key is test_row_0/A:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:51,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741863_1039 (size=16781) 2024-11-09T17:25:51,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173211512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173211512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173211514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173211516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,537 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173211619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173211620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173211621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173211623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173211649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:51,692 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173211824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173211826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173211826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173211830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,846 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:51,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:51,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:51,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:51,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e8c5c32863d4f7b842548fbed2d83b6 2024-11-09T17:25:51,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/00032ee55e104164b78e41f672943d14 is 50, key is test_row_0/B:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:51,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741864_1040 (size=12101) 2024-11-09T17:25:52,000 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:52,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173212129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173212132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173212132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173212134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,154 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:52,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/00032ee55e104164b78e41f672943d14 2024-11-09T17:25:52,310 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] handler.RSProcedureHandler(58): pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=19 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=19 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:52,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/cd82d06ed39d4183859dbb8d4f243635 is 50, key is test_row_0/C:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:52,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741865_1041 (size=12101) 2024-11-09T17:25:52,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/cd82d06ed39d4183859dbb8d4f243635 2024-11-09T17:25:52,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e8c5c32863d4f7b842548fbed2d83b6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6 2024-11-09T17:25:52,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6, entries=250, sequenceid=130, filesize=16.4 K 2024-11-09T17:25:52,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/00032ee55e104164b78e41f672943d14 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14 2024-11-09T17:25:52,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14, entries=150, sequenceid=130, filesize=11.8 K 2024-11-09T17:25:52,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/cd82d06ed39d4183859dbb8d4f243635 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635 2024-11-09T17:25:52,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635, entries=150, sequenceid=130, filesize=11.8 K 2024-11-09T17:25:52,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for d0b59f7cee8e92155959d9919114a0a2 in 934ms, sequenceid=130, compaction requested=true 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:52,395 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:52,395 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:52,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:52,397 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:52,397 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:52,398 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:52,398 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:52,398 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,398 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3927dcd5e6584f93bc5642c7e53758a9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=40.0 K 2024-11-09T17:25:52,398 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,398 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/65a10973659248c28c5ae99074a6907b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.5 K 2024-11-09T17:25:52,400 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3927dcd5e6584f93bc5642c7e53758a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:52,400 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 65a10973659248c28c5ae99074a6907b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:52,401 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting b74da3fd5888449586053afebece3093, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731173150507 2024-11-09T17:25:52,401 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1855c3a3f68a4137bff61cb4fd4427c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731173150507 2024-11-09T17:25:52,402 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 00032ee55e104164b78e41f672943d14, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:52,402 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e8c5c32863d4f7b842548fbed2d83b6, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:52,434 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:52,435 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/115a49c5cdbd4aa9b3caf659ae351f7c is 50, key is test_row_0/B:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:52,439 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:52,439 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b47a759339464ba588216a53bd03c4a0 is 50, key is test_row_0/A:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:52,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741867_1043 (size=12409) 2024-11-09T17:25:52,472 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=19 2024-11-09T17:25:52,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T17:25:52,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:52,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:52,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:52,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:52,488 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b47a759339464ba588216a53bd03c4a0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b47a759339464ba588216a53bd03c4a0 2024-11-09T17:25:52,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e31e4d325d140679c469f99af4f5515 is 50, key is test_row_0/A:col10/1731173151511/Put/seqid=0 2024-11-09T17:25:52,501 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into b47a759339464ba588216a53bd03c4a0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:52,501 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:52,502 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173152395; duration=0sec 2024-11-09T17:25:52,502 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:52,502 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:52,502 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:52,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741866_1042 (size=12409) 2024-11-09T17:25:52,512 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:52,512 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:52,512 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:52,512 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/77003a63983044d3ae706268ac6c39d9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=35.5 K 2024-11-09T17:25:52,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 77003a63983044d3ae706268ac6c39d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731173149306 2024-11-09T17:25:52,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting fbe7dc89f5db457ca8485cd16ddcafb3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731173150507 2024-11-09T17:25:52,520 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cd82d06ed39d4183859dbb8d4f243635, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:52,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741868_1044 (size=12151) 2024-11-09T17:25:52,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e31e4d325d140679c469f99af4f5515 2024-11-09T17:25:52,541 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:52,542 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/e1a9f3a842c345029440870f21c55ff1 is 50, key is test_row_0/C:col10/1731173150833/Put/seqid=0 2024-11-09T17:25:52,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3b168af4410c41f894c01e5a09f517f7 is 50, key is test_row_0/B:col10/1731173151511/Put/seqid=0 2024-11-09T17:25:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741870_1046 (size=12151) 2024-11-09T17:25:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741869_1045 (size=12409) 2024-11-09T17:25:52,568 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3b168af4410c41f894c01e5a09f517f7 2024-11-09T17:25:52,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/2c8f0e6cbf1745f7ba92a21f87614c8f is 50, key is test_row_0/C:col10/1731173151511/Put/seqid=0 2024-11-09T17:25:52,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741871_1047 (size=12151) 2024-11-09T17:25:52,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:52,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173212647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173212648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173212652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173212653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173212660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173212754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173212754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173212758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173212758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,913 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/115a49c5cdbd4aa9b3caf659ae351f7c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/115a49c5cdbd4aa9b3caf659ae351f7c 2024-11-09T17:25:52,923 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 115a49c5cdbd4aa9b3caf659ae351f7c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:52,923 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:52,923 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173152395; duration=0sec 2024-11-09T17:25:52,924 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:52,924 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173212958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173212960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173212963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173212964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:52,975 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/e1a9f3a842c345029440870f21c55ff1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/e1a9f3a842c345029440870f21c55ff1 2024-11-09T17:25:52,983 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into e1a9f3a842c345029440870f21c55ff1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:52,983 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:52,983 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173152395; duration=0sec 2024-11-09T17:25:52,983 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:52,983 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:52,994 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/2c8f0e6cbf1745f7ba92a21f87614c8f 2024-11-09T17:25:53,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/2e31e4d325d140679c469f99af4f5515 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515 2024-11-09T17:25:53,011 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T17:25:53,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3b168af4410c41f894c01e5a09f517f7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7 2024-11-09T17:25:53,021 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T17:25:53,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/2c8f0e6cbf1745f7ba92a21f87614c8f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f 2024-11-09T17:25:53,032 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T17:25:53,034 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for d0b59f7cee8e92155959d9919114a0a2 in 560ms, sequenceid=156, compaction requested=false 2024-11-09T17:25:53,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-09T17:25:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=19 2024-11-09T17:25:53,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=19, resume processing ppid=18 2024-11-09T17:25:53,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, ppid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9630 sec 2024-11-09T17:25:53,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=18, table=TestAcidGuarantees in 1.9750 sec 2024-11-09T17:25:53,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=18 2024-11-09T17:25:53,179 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 18 completed 2024-11-09T17:25:53,181 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:53,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees 2024-11-09T17:25:53,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:53,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:53,187 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=20, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:53,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:53,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:25:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:53,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:53,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:53,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:53,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/cf96207fadde462d8479c1ca6735ebb3 is 50, key is test_row_0/A:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:53,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741872_1048 (size=9757) 2024-11-09T17:25:53,342 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173213337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173213337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173213340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173213339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173213445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173213446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173213446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173213447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:53,497 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,652 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:53,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173213650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173213652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173213653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173213654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/cf96207fadde462d8479c1ca6735ebb3 2024-11-09T17:25:53,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/81ed6c5fb3724d6493b915d4424b0a14 is 50, key is test_row_0/B:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:53,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741873_1049 (size=9757) 2024-11-09T17:25:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:53,813 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:53,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173213956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173213956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173213956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:53,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173213964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,967 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:53,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:53,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:53,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,121 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/81ed6c5fb3724d6493b915d4424b0a14 2024-11-09T17:25:54,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/1ac3331961b34a65a4e926a5d46e0cc8 is 50, key is test_row_0/C:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:54,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741874_1050 (size=9757) 2024-11-09T17:25:54,275 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:54,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:54,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:54,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,460 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173214459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] handler.RSProcedureHandler(58): pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=21 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=21 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:54,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173214460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173214461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173214466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/1ac3331961b34a65a4e926a5d46e0cc8 2024-11-09T17:25:54,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/cf96207fadde462d8479c1ca6735ebb3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3 2024-11-09T17:25:54,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3, entries=100, sequenceid=172, filesize=9.5 K 2024-11-09T17:25:54,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/81ed6c5fb3724d6493b915d4424b0a14 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14 2024-11-09T17:25:54,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14, entries=100, sequenceid=172, filesize=9.5 K 2024-11-09T17:25:54,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/1ac3331961b34a65a4e926a5d46e0cc8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8 2024-11-09T17:25:54,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8, entries=100, sequenceid=172, filesize=9.5 K 2024-11-09T17:25:54,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d0b59f7cee8e92155959d9919114a0a2 in 1324ms, sequenceid=172, compaction requested=true 2024-11-09T17:25:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:54,592 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:54,592 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:54,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:54,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:54,594 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:54,594 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:54,594 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,594 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b47a759339464ba588216a53bd03c4a0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=33.5 K 2024-11-09T17:25:54,595 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b47a759339464ba588216a53bd03c4a0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:54,595 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:54,595 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:54,595 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,595 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/115a49c5cdbd4aa9b3caf659ae351f7c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=33.5 K 2024-11-09T17:25:54,596 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2e31e4d325d140679c469f99af4f5515, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173151509 2024-11-09T17:25:54,596 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 115a49c5cdbd4aa9b3caf659ae351f7c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:54,596 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cf96207fadde462d8479c1ca6735ebb3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173153265 2024-11-09T17:25:54,597 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 3b168af4410c41f894c01e5a09f517f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173151509 2024-11-09T17:25:54,597 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 81ed6c5fb3724d6493b915d4424b0a14, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173153265 2024-11-09T17:25:54,609 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:54,609 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:54,610 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/328745798efa4baf9ce909d5404fa86a is 50, key is test_row_0/B:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:54,610 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/be56424edf5343fc8bbcf17ec2822291 is 50, key is test_row_0/A:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:54,613 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=21 2024-11-09T17:25:54,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,615 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:54,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:54,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/f40cd4a790f54ee19eed8a11a47d9798 is 50, key is test_row_0/A:col10/1731173153338/Put/seqid=0 2024-11-09T17:25:54,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741875_1051 (size=12561) 2024-11-09T17:25:54,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741876_1052 (size=12561) 2024-11-09T17:25:54,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:54,687 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/be56424edf5343fc8bbcf17ec2822291 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/be56424edf5343fc8bbcf17ec2822291 2024-11-09T17:25:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741877_1053 (size=12151) 2024-11-09T17:25:54,697 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/f40cd4a790f54ee19eed8a11a47d9798 2024-11-09T17:25:54,706 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into be56424edf5343fc8bbcf17ec2822291(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:54,708 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:54,708 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173154591; duration=0sec 2024-11-09T17:25:54,709 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:54,709 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:54,709 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:54,711 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:54,712 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:54,712 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:54,712 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/e1a9f3a842c345029440870f21c55ff1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=33.5 K 2024-11-09T17:25:54,713 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e1a9f3a842c345029440870f21c55ff1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731173150833 2024-11-09T17:25:54,714 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2c8f0e6cbf1745f7ba92a21f87614c8f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173151509 2024-11-09T17:25:54,715 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1ac3331961b34a65a4e926a5d46e0cc8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173153265 2024-11-09T17:25:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/bc7161b0701d4c2e9631c0ed29893575 is 50, key is test_row_0/B:col10/1731173153338/Put/seqid=0 2024-11-09T17:25:54,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173214731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:54,753 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:54,754 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/a01757fac1314afeb70d5ae3697eb101 is 50, key is test_row_0/C:col10/1731173153265/Put/seqid=0 2024-11-09T17:25:54,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741878_1054 (size=12151) 2024-11-09T17:25:54,757 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/bc7161b0701d4c2e9631c0ed29893575 2024-11-09T17:25:54,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c476e5a10a7047479ab6356db61ed8f7 is 50, key is test_row_0/C:col10/1731173153338/Put/seqid=0 2024-11-09T17:25:54,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741879_1055 (size=12561) 2024-11-09T17:25:54,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741880_1056 (size=12151) 2024-11-09T17:25:54,807 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/a01757fac1314afeb70d5ae3697eb101 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/a01757fac1314afeb70d5ae3697eb101 2024-11-09T17:25:54,819 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into a01757fac1314afeb70d5ae3697eb101(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:54,819 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:54,819 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173154592; duration=0sec 2024-11-09T17:25:54,819 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:54,819 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:54,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173214838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173215040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,070 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/328745798efa4baf9ce909d5404fa86a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/328745798efa4baf9ce909d5404fa86a 2024-11-09T17:25:55,081 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 328745798efa4baf9ce909d5404fa86a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:55,081 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:55,081 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173154592; duration=0sec 2024-11-09T17:25:55,081 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:55,081 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:55,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c476e5a10a7047479ab6356db61ed8f7 2024-11-09T17:25:55,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/f40cd4a790f54ee19eed8a11a47d9798 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798 2024-11-09T17:25:55,224 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:25:55,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/bc7161b0701d4c2e9631c0ed29893575 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575 2024-11-09T17:25:55,232 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:25:55,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/c476e5a10a7047479ab6356db61ed8f7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7 2024-11-09T17:25:55,241 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:25:55,243 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d0b59f7cee8e92155959d9919114a0a2 in 628ms, sequenceid=195, compaction requested=false 2024-11-09T17:25:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=21}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=21 2024-11-09T17:25:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=21 2024-11-09T17:25:55,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=21, resume processing ppid=20 2024-11-09T17:25:55,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, ppid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-11-09T17:25:55,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=20, table=TestAcidGuarantees in 2.0660 sec 2024-11-09T17:25:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=20 2024-11-09T17:25:55,288 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 20 completed 2024-11-09T17:25:55,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:55,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees 2024-11-09T17:25:55,293 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:55,295 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=22, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:55,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:55,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:55,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/63c375188beb4c6cadb02429d489fbb6 is 50, key is test_row_0/A:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741881_1057 (size=14541) 2024-11-09T17:25:55,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/63c375188beb4c6cadb02429d489fbb6 2024-11-09T17:25:55,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/8528a7a2873b447e9ee0a42db62a4213 is 50, key is test_row_0/B:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741882_1058 (size=12151) 2024-11-09T17:25:55,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173215426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,449 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-09T17:25:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173215462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173215472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173215473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173215473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173215529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:55,603 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-09T17:25:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:55,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173215732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,756 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-09T17:25:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] handler.RSProcedureHandler(58): pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=23 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=23 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:55,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/8528a7a2873b447e9ee0a42db62a4213 2024-11-09T17:25:55,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/889220142fd5422ca1ede35fc85f1eda is 50, key is test_row_0/C:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741883_1059 (size=12151) 2024-11-09T17:25:55,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/889220142fd5422ca1ede35fc85f1eda 2024-11-09T17:25:55,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/63c375188beb4c6cadb02429d489fbb6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6 2024-11-09T17:25:55,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6, entries=200, sequenceid=212, filesize=14.2 K 2024-11-09T17:25:55,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/8528a7a2873b447e9ee0a42db62a4213 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213 2024-11-09T17:25:55,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213, entries=150, sequenceid=212, filesize=11.9 K 2024-11-09T17:25:55,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/889220142fd5422ca1ede35fc85f1eda as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda 2024-11-09T17:25:55,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda, entries=150, sequenceid=212, filesize=11.9 K 2024-11-09T17:25:55,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d0b59f7cee8e92155959d9919114a0a2 in 524ms, sequenceid=212, compaction requested=true 2024-11-09T17:25:55,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:55,868 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:55,870 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:55,870 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:55,870 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,870 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/be56424edf5343fc8bbcf17ec2822291, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=38.3 K 2024-11-09T17:25:55,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:55,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:55,871 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:55,871 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting be56424edf5343fc8bbcf17ec2822291, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173151509 2024-11-09T17:25:55,871 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f40cd4a790f54ee19eed8a11a47d9798, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173153335 2024-11-09T17:25:55,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:55,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:55,871 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 63c375188beb4c6cadb02429d489fbb6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154684 2024-11-09T17:25:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:55,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:55,872 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:55,872 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:55,872 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,873 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/328745798efa4baf9ce909d5404fa86a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.0 K 2024-11-09T17:25:55,873 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 328745798efa4baf9ce909d5404fa86a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173151509 2024-11-09T17:25:55,875 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting bc7161b0701d4c2e9631c0ed29893575, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173153335 2024-11-09T17:25:55,875 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8528a7a2873b447e9ee0a42db62a4213, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154721 2024-11-09T17:25:55,891 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:55,892 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/9d947a8336814b76bf3934186856e399 is 50, key is test_row_0/A:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,892 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#46 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:55,893 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe is 50, key is test_row_0/B:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:55,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741884_1060 (size=12663) 2024-11-09T17:25:55,910 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741885_1061 (size=12663) 2024-11-09T17:25:55,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=23 2024-11-09T17:25:55,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,913 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:55,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:55,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b7990a79263c402aa28d40c67b5f3a22 is 50, key is test_row_0/A:col10/1731173155417/Put/seqid=0 2024-11-09T17:25:55,925 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/9d947a8336814b76bf3934186856e399 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9d947a8336814b76bf3934186856e399 2024-11-09T17:25:55,941 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 9d947a8336814b76bf3934186856e399(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:55,941 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:55,941 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173155868; duration=0sec 2024-11-09T17:25:55,942 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:55,942 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:55,942 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:55,943 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:55,944 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:55,944 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:55,944 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/a01757fac1314afeb70d5ae3697eb101, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.0 K 2024-11-09T17:25:55,945 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting a01757fac1314afeb70d5ae3697eb101, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173151509 2024-11-09T17:25:55,945 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting c476e5a10a7047479ab6356db61ed8f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173153335 2024-11-09T17:25:55,946 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 889220142fd5422ca1ede35fc85f1eda, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154721 2024-11-09T17:25:55,961 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:55,962 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/587bdd983b63440c8cd1c6180096cabd is 50, key is test_row_0/C:col10/1731173154721/Put/seqid=0 2024-11-09T17:25:55,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741886_1062 (size=12151) 2024-11-09T17:25:55,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741887_1063 (size=12663) 2024-11-09T17:25:55,991 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/587bdd983b63440c8cd1c6180096cabd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/587bdd983b63440c8cd1c6180096cabd 2024-11-09T17:25:56,000 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 587bdd983b63440c8cd1c6180096cabd(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:56,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:56,000 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173155871; duration=0sec 2024-11-09T17:25:56,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:56,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:56,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:56,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173216072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:56,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173216175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:56,318 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe 2024-11-09T17:25:56,325 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 6b2ee7daf22a4dc385f7b5bdc1fa8cbe(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:56,325 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:56,325 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173155870; duration=0sec 2024-11-09T17:25:56,326 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:56,326 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:56,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b7990a79263c402aa28d40c67b5f3a22 2024-11-09T17:25:56,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:56,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173216381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:56,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/7cad540176024fa59e0f9e2172c49978 is 50, key is test_row_0/B:col10/1731173155417/Put/seqid=0 2024-11-09T17:25:56,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:56,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741888_1064 (size=12151) 2024-11-09T17:25:56,407 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/7cad540176024fa59e0f9e2172c49978 2024-11-09T17:25:56,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8f6d8cd75d06441ba2da7b854a215731 is 50, key is test_row_0/C:col10/1731173155417/Put/seqid=0 2024-11-09T17:25:56,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741889_1065 (size=12151) 2024-11-09T17:25:56,422 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8f6d8cd75d06441ba2da7b854a215731 2024-11-09T17:25:56,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b7990a79263c402aa28d40c67b5f3a22 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22 2024-11-09T17:25:56,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22, entries=150, sequenceid=234, filesize=11.9 K 2024-11-09T17:25:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/7cad540176024fa59e0f9e2172c49978 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978 2024-11-09T17:25:56,444 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978, entries=150, sequenceid=234, filesize=11.9 K 2024-11-09T17:25:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8f6d8cd75d06441ba2da7b854a215731 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731 2024-11-09T17:25:56,451 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731, entries=150, sequenceid=234, filesize=11.9 K 2024-11-09T17:25:56,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d0b59f7cee8e92155959d9919114a0a2 in 540ms, sequenceid=234, compaction requested=false 2024-11-09T17:25:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:56,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-11-09T17:25:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=23 2024-11-09T17:25:56,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=23, resume processing ppid=22 2024-11-09T17:25:56,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, ppid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1580 sec 2024-11-09T17:25:56,457 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=22, table=TestAcidGuarantees in 1.1640 sec 2024-11-09T17:25:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:56,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:25:56,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:56,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/adcc607387be4eb2b63a4d7555e446c2 is 50, key is test_row_0/A:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741890_1066 (size=14541) 2024-11-09T17:25:56,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/adcc607387be4eb2b63a4d7555e446c2 2024-11-09T17:25:56,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/e32c516dd0134bd5b87b6ac354725cfc is 50, key is test_row_0/B:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173216770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:56,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741891_1067 (size=12151) 2024-11-09T17:25:56,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/e32c516dd0134bd5b87b6ac354725cfc 2024-11-09T17:25:56,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ed7841695d0a4b2cb6c566ad7ae2d312 is 50, key is test_row_0/C:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741892_1068 (size=12151) 2024-11-09T17:25:56,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ed7841695d0a4b2cb6c566ad7ae2d312 2024-11-09T17:25:56,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/adcc607387be4eb2b63a4d7555e446c2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2 2024-11-09T17:25:56,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2, entries=200, sequenceid=252, filesize=14.2 K 2024-11-09T17:25:56,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/e32c516dd0134bd5b87b6ac354725cfc as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc 2024-11-09T17:25:56,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T17:25:56,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ed7841695d0a4b2cb6c566ad7ae2d312 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312 2024-11-09T17:25:56,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312, entries=150, sequenceid=252, filesize=11.9 K 2024-11-09T17:25:56,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d0b59f7cee8e92155959d9919114a0a2 in 179ms, sequenceid=252, compaction requested=true 2024-11-09T17:25:56,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:56,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:56,865 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:56,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:56,865 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:56,867 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:56,867 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:56,867 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:56,867 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.1 K 2024-11-09T17:25:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:56,868 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:56,868 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 6b2ee7daf22a4dc385f7b5bdc1fa8cbe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154721 2024-11-09T17:25:56,868 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:56,868 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:56,869 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9d947a8336814b76bf3934186856e399, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=38.4 K 2024-11-09T17:25:56,869 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7cad540176024fa59e0f9e2172c49978, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731173155397 2024-11-09T17:25:56,870 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9d947a8336814b76bf3934186856e399, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154721 2024-11-09T17:25:56,870 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e32c516dd0134bd5b87b6ac354725cfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:56,870 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b7990a79263c402aa28d40c67b5f3a22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731173155397 2024-11-09T17:25:56,871 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting adcc607387be4eb2b63a4d7555e446c2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:56,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:25:56,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:56,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:56,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:56,884 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:56,887 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:56,888 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/ce7feedf5144445088fe89f14ff6a7e5 is 50, key is test_row_0/B:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,897 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#55 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:56,897 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/832843c92c6a4625a20b7ce4bb2f4d18 is 50, key is test_row_0/A:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/98b3a3ebda704a2bb406faa8bba34484 is 50, key is test_row_0/A:col10/1731173156882/Put/seqid=0 2024-11-09T17:25:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173216925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:56,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741893_1069 (size=12765) 2024-11-09T17:25:56,938 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/ce7feedf5144445088fe89f14ff6a7e5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ce7feedf5144445088fe89f14ff6a7e5 2024-11-09T17:25:56,948 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into ce7feedf5144445088fe89f14ff6a7e5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:56,948 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:56,948 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173156865; duration=0sec 2024-11-09T17:25:56,948 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:56,948 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:56,948 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:56,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741894_1070 (size=12765) 2024-11-09T17:25:56,951 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:56,951 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:56,952 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:56,952 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/587bdd983b63440c8cd1c6180096cabd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.1 K 2024-11-09T17:25:56,953 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 587bdd983b63440c8cd1c6180096cabd, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731173154721 2024-11-09T17:25:56,953 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8f6d8cd75d06441ba2da7b854a215731, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731173155397 2024-11-09T17:25:56,953 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ed7841695d0a4b2cb6c566ad7ae2d312, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:56,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741895_1071 (size=12301) 2024-11-09T17:25:56,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/98b3a3ebda704a2bb406faa8bba34484 2024-11-09T17:25:56,970 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:56,971 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/550712b8cafb409981baaecf618b560f is 50, key is test_row_0/C:col10/1731173156054/Put/seqid=0 2024-11-09T17:25:56,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741896_1072 (size=12765) 2024-11-09T17:25:56,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/932e1ed5cf504b39b7f1797470545fec is 50, key is test_row_0/B:col10/1731173156882/Put/seqid=0 2024-11-09T17:25:57,002 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/550712b8cafb409981baaecf618b560f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/550712b8cafb409981baaecf618b560f 2024-11-09T17:25:57,011 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 550712b8cafb409981baaecf618b560f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:57,011 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:57,012 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173156867; duration=0sec 2024-11-09T17:25:57,012 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:57,012 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:57,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741897_1073 (size=12301) 2024-11-09T17:25:57,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/932e1ed5cf504b39b7f1797470545fec 2024-11-09T17:25:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173217030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/69419ea7a4a845fe88a7c923d9e65009 is 50, key is test_row_0/C:col10/1731173156882/Put/seqid=0 2024-11-09T17:25:57,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741898_1074 (size=12301) 2024-11-09T17:25:57,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/69419ea7a4a845fe88a7c923d9e65009 2024-11-09T17:25:57,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/98b3a3ebda704a2bb406faa8bba34484 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484 2024-11-09T17:25:57,080 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484, entries=150, sequenceid=275, filesize=12.0 K 2024-11-09T17:25:57,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/932e1ed5cf504b39b7f1797470545fec as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec 2024-11-09T17:25:57,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec, entries=150, sequenceid=275, filesize=12.0 K 2024-11-09T17:25:57,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/69419ea7a4a845fe88a7c923d9e65009 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009 2024-11-09T17:25:57,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009, entries=150, sequenceid=275, filesize=12.0 K 2024-11-09T17:25:57,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d0b59f7cee8e92155959d9919114a0a2 in 227ms, sequenceid=275, compaction requested=false 2024-11-09T17:25:57,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:57,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:57,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:57,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/bf182540ed3b4cd49540050ac8a70f49 is 50, key is test_row_0/A:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:57,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741899_1075 (size=12301) 2024-11-09T17:25:57,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/bf182540ed3b4cd49540050ac8a70f49 2024-11-09T17:25:57,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 is 50, key is test_row_0/B:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:57,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741900_1076 (size=12301) 2024-11-09T17:25:57,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173217315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,357 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/832843c92c6a4625a20b7ce4bb2f4d18 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/832843c92c6a4625a20b7ce4bb2f4d18 2024-11-09T17:25:57,364 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 832843c92c6a4625a20b7ce4bb2f4d18(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:57,365 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:57,365 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173156865; duration=0sec 2024-11-09T17:25:57,365 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:57,365 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=22 2024-11-09T17:25:57,399 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 22 completed 2024-11-09T17:25:57,400 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees 2024-11-09T17:25:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:57,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:57,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=24, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:57,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:57,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173217418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173217476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,478 DEBUG [Thread-155 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4141 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:25:57,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173217480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,482 DEBUG [Thread-151 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4145 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:25:57,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173217484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173217484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,485 DEBUG [Thread-157 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:25:57,486 DEBUG [Thread-153 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:25:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:57,554 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-09T17:25:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173217624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 2024-11-09T17:25:57,687 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/47c847d4da1b48cca7b1d125c56e4c4b is 50, key is test_row_0/C:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:57,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741901_1077 (size=12301) 2024-11-09T17:25:57,709 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-09T17:25:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:57,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,861 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:57,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-09T17:25:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:57,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:57,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:57,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173217927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:58,014 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:58,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-09T17:25:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] handler.RSProcedureHandler(58): pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:58,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=25 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=25 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:58,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/47c847d4da1b48cca7b1d125c56e4c4b 2024-11-09T17:25:58,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/bf182540ed3b4cd49540050ac8a70f49 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49 2024-11-09T17:25:58,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T17:25:58,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 2024-11-09T17:25:58,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T17:25:58,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/47c847d4da1b48cca7b1d125c56e4c4b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b 2024-11-09T17:25:58,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b, entries=150, sequenceid=291, filesize=12.0 K 2024-11-09T17:25:58,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d0b59f7cee8e92155959d9919114a0a2 in 907ms, sequenceid=291, compaction requested=true 2024-11-09T17:25:58,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:58,144 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:58,145 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:25:58,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:58,146 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:58,146 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:25:58,146 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,146 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/832843c92c6a4625a20b7ce4bb2f4d18, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.5 K 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 832843c92c6a4625a20b7ce4bb2f4d18, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:25:58,147 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 98b3a3ebda704a2bb406faa8bba34484, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731173156768 2024-11-09T17:25:58,147 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ce7feedf5144445088fe89f14ff6a7e5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.5 K 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting bf182540ed3b4cd49540050ac8a70f49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:25:58,147 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ce7feedf5144445088fe89f14ff6a7e5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:58,148 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 932e1ed5cf504b39b7f1797470545fec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731173156768 2024-11-09T17:25:58,148 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a5d6fb67084b4bc59fe4b1ff7c9f8ca0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:25:58,158 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:58,158 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:58,158 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/8e3db8de376f473fb97bcbc7fdd1e9b1 is 50, key is test_row_0/B:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:58,158 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3758be1755a5489abb6bb2137d7397a7 is 50, key is test_row_0/A:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:58,167 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:58,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=25 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,168 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:58,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:58,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741902_1078 (size=13017) 2024-11-09T17:25:58,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741903_1079 (size=13017) 2024-11-09T17:25:58,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b18a28ee15944e3a9f93e71f749367c2 is 50, key is test_row_0/A:col10/1731173157302/Put/seqid=0 2024-11-09T17:25:58,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741904_1080 (size=12301) 2024-11-09T17:25:58,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:58,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:58,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173218470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:58,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:58,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173218574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:58,586 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3758be1755a5489abb6bb2137d7397a7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3758be1755a5489abb6bb2137d7397a7 2024-11-09T17:25:58,588 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/8e3db8de376f473fb97bcbc7fdd1e9b1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8e3db8de376f473fb97bcbc7fdd1e9b1 2024-11-09T17:25:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b18a28ee15944e3a9f93e71f749367c2 2024-11-09T17:25:58,600 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 8e3db8de376f473fb97bcbc7fdd1e9b1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:58,600 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:58,601 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173158145; duration=0sec 2024-11-09T17:25:58,601 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:25:58,601 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:25:58,601 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:25:58,604 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:25:58,604 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:25:58,604 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:58,604 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/550712b8cafb409981baaecf618b560f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.5 K 2024-11-09T17:25:58,604 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 3758be1755a5489abb6bb2137d7397a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:58,605 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:58,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 is 50, key is test_row_0/B:col10/1731173157302/Put/seqid=0 2024-11-09T17:25:58,605 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173158144; duration=0sec 2024-11-09T17:25:58,605 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:58,605 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:25:58,605 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 550712b8cafb409981baaecf618b560f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731173156054 2024-11-09T17:25:58,606 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 69419ea7a4a845fe88a7c923d9e65009, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731173156768 2024-11-09T17:25:58,606 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 47c847d4da1b48cca7b1d125c56e4c4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:25:58,625 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:25:58,626 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/7cf32169c83f4fd4a36795e557dab3a7 is 50, key is test_row_0/C:col10/1731173156910/Put/seqid=0 2024-11-09T17:25:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741905_1081 (size=12301) 2024-11-09T17:25:58,630 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 2024-11-09T17:25:58,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741906_1082 (size=13017) 2024-11-09T17:25:58,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/0db0b51b75bf4b4caf4bbb654184cab8 is 50, key is test_row_0/C:col10/1731173157302/Put/seqid=0 2024-11-09T17:25:58,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741907_1083 (size=12301) 2024-11-09T17:25:58,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173218777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/7cf32169c83f4fd4a36795e557dab3a7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7cf32169c83f4fd4a36795e557dab3a7 2024-11-09T17:25:59,052 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 7cf32169c83f4fd4a36795e557dab3a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:25:59,052 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/0db0b51b75bf4b4caf4bbb654184cab8 2024-11-09T17:25:59,053 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:59,053 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173158145; duration=0sec 2024-11-09T17:25:59,053 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:25:59,053 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:25:59,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/b18a28ee15944e3a9f93e71f749367c2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2 2024-11-09T17:25:59,067 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2, entries=150, sequenceid=314, filesize=12.0 K 2024-11-09T17:25:59,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 2024-11-09T17:25:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186, entries=150, sequenceid=314, filesize=12.0 K 2024-11-09T17:25:59,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/0db0b51b75bf4b4caf4bbb654184cab8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8 2024-11-09T17:25:59,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:59,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173219082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,083 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8, entries=150, sequenceid=314, filesize=12.0 K 2024-11-09T17:25:59,084 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d0b59f7cee8e92155959d9919114a0a2 in 916ms, sequenceid=314, compaction requested=false 2024-11-09T17:25:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:25:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-11-09T17:25:59,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-11-09T17:25:59,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-09T17:25:59,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6830 sec 2024-11-09T17:25:59,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=24, table=TestAcidGuarantees in 1.6890 sec 2024-11-09T17:25:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-09T17:25:59,506 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 24 completed 2024-11-09T17:25:59,507 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:25:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees 2024-11-09T17:25:59,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:25:59,510 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=26, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:25:59,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:25:59,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:25:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:25:59,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:25:59,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:25:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:25:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:25:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:25:59,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/81e44ffe46f547b69cc7c62235932ab0 is 50, key is test_row_0/A:col10/1731173159584/Put/seqid=0 2024-11-09T17:25:59,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741908_1084 (size=14741) 2024-11-09T17:25:59,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/81e44ffe46f547b69cc7c62235932ab0 2024-11-09T17:25:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:25:59,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4f4f27e61bea42a7bf4522995078e2eb is 50, key is test_row_0/B:col10/1731173159584/Put/seqid=0 2024-11-09T17:25:59,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741909_1085 (size=12301) 2024-11-09T17:25:59,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4f4f27e61bea42a7bf4522995078e2eb 2024-11-09T17:25:59,663 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-09T17:25:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ffa50c776e22452580227d19f583d300 is 50, key is test_row_0/C:col10/1731173159584/Put/seqid=0 2024-11-09T17:25:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741910_1086 (size=12301) 2024-11-09T17:25:59,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:59,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173219675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:59,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173219777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:25:59,820 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-09T17:25:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,973 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:25:59,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-09T17:25:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:25:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:25:59,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] handler.RSProcedureHandler(58): pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=27 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=27 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:25:59,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:25:59,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173219981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:00,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ffa50c776e22452580227d19f583d300 2024-11-09T17:26:00,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/81e44ffe46f547b69cc7c62235932ab0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0 2024-11-09T17:26:00,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0, entries=200, sequenceid=332, filesize=14.4 K 2024-11-09T17:26:00,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/4f4f27e61bea42a7bf4522995078e2eb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb 2024-11-09T17:26:00,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T17:26:00,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/ffa50c776e22452580227d19f583d300 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300 2024-11-09T17:26:00,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T17:26:00,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d0b59f7cee8e92155959d9919114a0a2 in 517ms, sequenceid=332, compaction requested=true 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:00,103 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:00,103 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:00,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:00,104 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:00,104 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:26:00,105 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:00,105 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8e3db8de376f473fb97bcbc7fdd1e9b1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.7 K 2024-11-09T17:26:00,105 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:00,105 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:26:00,105 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:00,105 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3758be1755a5489abb6bb2137d7397a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=39.1 K 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3758be1755a5489abb6bb2137d7397a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8e3db8de376f473fb97bcbc7fdd1e9b1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting eeb83c9ed3b442cdb2bdc1a4ed29a186, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731173157289 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b18a28ee15944e3a9f93e71f749367c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731173157289 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 81e44ffe46f547b69cc7c62235932ab0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:00,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4f4f27e61bea42a7bf4522995078e2eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:00,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:26:00,119 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:00,120 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/564ba6134d62454fa9d20d7ac6c9f7b5 is 50, key is test_row_0/B:col10/1731173159584/Put/seqid=0 2024-11-09T17:26:00,125 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:00,126 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/40074587e9c94e2abb6e9f2073976c42 is 50, key is test_row_0/A:col10/1731173159584/Put/seqid=0 2024-11-09T17:26:00,126 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:00,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=27 2024-11-09T17:26:00,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:00,127 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741911_1087 (size=13119) 2024-11-09T17:26:00,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741912_1088 (size=13119) 2024-11-09T17:26:00,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/750c29c88946469aa10be8ed1babe131 is 50, key is test_row_0/A:col10/1731173159654/Put/seqid=0 2024-11-09T17:26:00,145 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/40074587e9c94e2abb6e9f2073976c42 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/40074587e9c94e2abb6e9f2073976c42 2024-11-09T17:26:00,145 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/564ba6134d62454fa9d20d7ac6c9f7b5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/564ba6134d62454fa9d20d7ac6c9f7b5 2024-11-09T17:26:00,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741913_1089 (size=12301) 2024-11-09T17:26:00,149 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/750c29c88946469aa10be8ed1babe131 2024-11-09T17:26:00,155 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 564ba6134d62454fa9d20d7ac6c9f7b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:00,155 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:00,155 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173160103; duration=0sec 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:00,156 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 40074587e9c94e2abb6e9f2073976c42(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:00,156 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173160103; duration=0sec 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:00,156 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:26:00,157 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:00,157 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:26:00,158 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:00,158 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7cf32169c83f4fd4a36795e557dab3a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.7 K 2024-11-09T17:26:00,159 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7cf32169c83f4fd4a36795e557dab3a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731173156910 2024-11-09T17:26:00,159 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0db0b51b75bf4b4caf4bbb654184cab8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731173157289 2024-11-09T17:26:00,160 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ffa50c776e22452580227d19f583d300, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:00,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/2cabee6f87bc4076a5a53f90252721a1 is 50, key is test_row_0/B:col10/1731173159654/Put/seqid=0 2024-11-09T17:26:00,170 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:00,171 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/eab74bc90e144d1b945eb64fa4c5dd90 is 50, key is test_row_0/C:col10/1731173159584/Put/seqid=0 2024-11-09T17:26:00,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741914_1090 (size=12301) 2024-11-09T17:26:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741915_1091 (size=13119) 2024-11-09T17:26:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:00,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:00,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173220314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:00,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173220416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:00,583 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/2cabee6f87bc4076a5a53f90252721a1 2024-11-09T17:26:00,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/7053dcb514e54981b96d1579c7e7118c is 50, key is test_row_0/C:col10/1731173159654/Put/seqid=0 2024-11-09T17:26:00,606 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/eab74bc90e144d1b945eb64fa4c5dd90 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/eab74bc90e144d1b945eb64fa4c5dd90 2024-11-09T17:26:00,613 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into eab74bc90e144d1b945eb64fa4c5dd90(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:00,614 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:00,614 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173160103; duration=0sec 2024-11-09T17:26:00,614 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:00,614 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:26:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:26:00,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173220621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:00,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741916_1092 (size=12301) 2024-11-09T17:26:00,633 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/7053dcb514e54981b96d1579c7e7118c 2024-11-09T17:26:00,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/750c29c88946469aa10be8ed1babe131 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131 2024-11-09T17:26:00,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T17:26:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/2cabee6f87bc4076a5a53f90252721a1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1 2024-11-09T17:26:00,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T17:26:00,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/7053dcb514e54981b96d1579c7e7118c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c 2024-11-09T17:26:00,676 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c, entries=150, sequenceid=353, filesize=12.0 K 2024-11-09T17:26:00,677 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for d0b59f7cee8e92155959d9919114a0a2 in 550ms, sequenceid=353, compaction requested=false 2024-11-09T17:26:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:00,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=27}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=27 2024-11-09T17:26:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=27 2024-11-09T17:26:00,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=27, resume processing ppid=26 2024-11-09T17:26:00,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, ppid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-09T17:26:00,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=26, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=26, table=TestAcidGuarantees in 1.1750 sec 2024-11-09T17:26:00,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:00,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:00,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:00,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/9116c6aeb2fb44138ae06fb8dde13e66 is 50, key is test_row_0/A:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:00,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741917_1093 (size=12301) 2024-11-09T17:26:00,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173220969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173221073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173221276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/9116c6aeb2fb44138ae06fb8dde13e66 2024-11-09T17:26:01,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc6286204fe34e21ac29f80cf2ae5b61 is 50, key is test_row_0/B:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:01,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741918_1094 (size=12301) 2024-11-09T17:26:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46714 deadline: 1731173221504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,506 DEBUG [Thread-155 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8168 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:26:01,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46758 deadline: 1731173221507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,508 DEBUG [Thread-153 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8168 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:26:01,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46768 deadline: 1731173221510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,512 DEBUG [Thread-151 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8174 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:26:01,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46704 deadline: 1731173221518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,519 DEBUG [Thread-157 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8180 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:26:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173221582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-09T17:26:01,615 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 26 completed 2024-11-09T17:26:01,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees 2024-11-09T17:26:01,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:01,617 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:01,618 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=28, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:01,618 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:01,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:01,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc6286204fe34e21ac29f80cf2ae5b61 2024-11-09T17:26:01,769 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-09T17:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:01,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:01,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8b79866593db45c897caa46f94e250da is 50, key is test_row_0/C:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:01,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741919_1095 (size=12301) 2024-11-09T17:26:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:01,925 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:01,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-09T17:26:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:01,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:01,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:02,078 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:02,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-09T17:26:02,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] handler.RSProcedureHandler(58): pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=29 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=29 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:02,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173222086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:02,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8b79866593db45c897caa46f94e250da 2024-11-09T17:26:02,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/9116c6aeb2fb44138ae06fb8dde13e66 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66 2024-11-09T17:26:02,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66, entries=150, sequenceid=372, filesize=12.0 K 2024-11-09T17:26:02,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc6286204fe34e21ac29f80cf2ae5b61 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61 2024-11-09T17:26:02,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61, entries=150, sequenceid=372, filesize=12.0 K 2024-11-09T17:26:02,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/8b79866593db45c897caa46f94e250da as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da 2024-11-09T17:26:02,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da, entries=150, sequenceid=372, filesize=12.0 K 2024-11-09T17:26:02,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for d0b59f7cee8e92155959d9919114a0a2 in 1283ms, sequenceid=372, compaction requested=true 2024-11-09T17:26:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:02,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:02,209 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:02,209 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:02,210 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:02,210 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:26:02,210 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,210 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/40074587e9c94e2abb6e9f2073976c42, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.8 K 2024-11-09T17:26:02,210 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:02,211 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:26:02,211 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,211 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/564ba6134d62454fa9d20d7ac6c9f7b5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.8 K 2024-11-09T17:26:02,211 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 40074587e9c94e2abb6e9f2073976c42, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:02,211 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 750c29c88946469aa10be8ed1babe131, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731173159654 2024-11-09T17:26:02,211 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 564ba6134d62454fa9d20d7ac6c9f7b5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:02,212 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9116c6aeb2fb44138ae06fb8dde13e66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:02,212 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2cabee6f87bc4076a5a53f90252721a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731173159654 2024-11-09T17:26:02,212 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting fc6286204fe34e21ac29f80cf2ae5b61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:02,222 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:02,223 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3b4e1f15b180482390e8a9ea60031cde is 50, key is test_row_0/A:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:02,227 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:02,227 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/5983db997e5145c48ea92b933ab3e810 is 50, key is test_row_0/B:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:02,232 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:02,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741920_1096 (size=13221) 2024-11-09T17:26:02,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=29 2024-11-09T17:26:02,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,233 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:02,241 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/3b4e1f15b180482390e8a9ea60031cde as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3b4e1f15b180482390e8a9ea60031cde 2024-11-09T17:26:02,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/7a7881753a38409396a49785de7f5bbb is 50, key is test_row_0/A:col10/1731173160959/Put/seqid=0 2024-11-09T17:26:02,249 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 3b4e1f15b180482390e8a9ea60031cde(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:02,249 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:02,249 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173162208; duration=0sec 2024-11-09T17:26:02,249 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:02,249 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:26:02,249 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:02,251 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:02,251 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:26:02,251 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:02,251 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/eab74bc90e144d1b945eb64fa4c5dd90, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.8 K 2024-11-09T17:26:02,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741921_1097 (size=13221) 2024-11-09T17:26:02,253 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting eab74bc90e144d1b945eb64fa4c5dd90, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731173158448 2024-11-09T17:26:02,254 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7053dcb514e54981b96d1579c7e7118c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731173159654 2024-11-09T17:26:02,254 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8b79866593db45c897caa46f94e250da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:02,260 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/5983db997e5145c48ea92b933ab3e810 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/5983db997e5145c48ea92b933ab3e810 2024-11-09T17:26:02,267 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 5983db997e5145c48ea92b933ab3e810(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:02,267 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:02,267 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173162209; duration=0sec 2024-11-09T17:26:02,267 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:02,267 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:26:02,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741922_1098 (size=12301) 2024-11-09T17:26:02,276 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:02,277 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/cba026849a3246988020b1c9ade3c63a is 50, key is test_row_0/C:col10/1731173160301/Put/seqid=0 2024-11-09T17:26:02,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741923_1099 (size=13221) 2024-11-09T17:26:02,293 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/cba026849a3246988020b1c9ade3c63a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cba026849a3246988020b1c9ade3c63a 2024-11-09T17:26:02,300 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into cba026849a3246988020b1c9ade3c63a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:02,300 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:02,300 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173162209; duration=0sec 2024-11-09T17:26:02,301 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:02,301 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:26:02,672 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/7a7881753a38409396a49785de7f5bbb 2024-11-09T17:26:02,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3bafe431968a4a1fadced3c049c97d27 is 50, key is test_row_0/B:col10/1731173160959/Put/seqid=0 2024-11-09T17:26:02,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741924_1100 (size=12301) 2024-11-09T17:26:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:03,088 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3bafe431968a4a1fadced3c049c97d27 2024-11-09T17:26:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:03,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:03,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/35b07ce48f1e4c6096ae909d19b7ef2e is 50, key is test_row_0/C:col10/1731173160959/Put/seqid=0 2024-11-09T17:26:03,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741925_1101 (size=12301) 2024-11-09T17:26:03,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:03,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173223130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:03,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173223233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173223436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:03,502 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/35b07ce48f1e4c6096ae909d19b7ef2e 2024-11-09T17:26:03,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/7a7881753a38409396a49785de7f5bbb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb 2024-11-09T17:26:03,513 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb, entries=150, sequenceid=392, filesize=12.0 K 2024-11-09T17:26:03,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/3bafe431968a4a1fadced3c049c97d27 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27 2024-11-09T17:26:03,520 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27, entries=150, sequenceid=392, filesize=12.0 K 2024-11-09T17:26:03,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/35b07ce48f1e4c6096ae909d19b7ef2e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e 2024-11-09T17:26:03,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e, entries=150, sequenceid=392, filesize=12.0 K 2024-11-09T17:26:03,528 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for d0b59f7cee8e92155959d9919114a0a2 in 1295ms, sequenceid=392, compaction requested=false 2024-11-09T17:26:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:03,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=29}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=29 2024-11-09T17:26:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=29 2024-11-09T17:26:03,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T17:26:03,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, ppid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9120 sec 2024-11-09T17:26:03,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=28, table=TestAcidGuarantees in 1.9160 sec 2024-11-09T17:26:03,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=28 2024-11-09T17:26:03,721 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 28 completed 2024-11-09T17:26:03,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees 2024-11-09T17:26:03,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:03,724 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:03,724 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=30, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:03,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:03,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:03,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:03,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:03,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1fda7a4be84143509166b58ff0f92057 is 50, key is test_row_0/A:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:03,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741926_1102 (size=14741) 2024-11-09T17:26:03,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 304 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173223796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:03,875 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:03,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:03,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:03,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:03,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:03,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:03,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:03,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:03,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 306 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173223899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,028 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:04,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:04,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 308 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173224102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1fda7a4be84143509166b58ff0f92057 2024-11-09T17:26:04,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/21583c56099e41bdabddd44dc7a72aa6 is 50, key is test_row_0/B:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:04,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741927_1103 (size=12301) 2024-11-09T17:26:04,181 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:04,334 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:04,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 310 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173224404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,487 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/21583c56099e41bdabddd44dc7a72aa6 2024-11-09T17:26:04,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/4d1ebe8029b0408fb26ae1325f40182c is 50, key is test_row_0/C:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:04,639 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] handler.RSProcedureHandler(58): pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=31 java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=31 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:04,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741928_1104 (size=12301) 2024-11-09T17:26:04,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/4d1ebe8029b0408fb26ae1325f40182c 2024-11-09T17:26:04,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1fda7a4be84143509166b58ff0f92057 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057 2024-11-09T17:26:04,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057, entries=200, sequenceid=412, filesize=14.4 K 2024-11-09T17:26:04,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/21583c56099e41bdabddd44dc7a72aa6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6 2024-11-09T17:26:04,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6, entries=150, sequenceid=412, filesize=12.0 K 2024-11-09T17:26:04,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/4d1ebe8029b0408fb26ae1325f40182c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c 2024-11-09T17:26:04,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c, entries=150, sequenceid=412, filesize=12.0 K 2024-11-09T17:26:04,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for d0b59f7cee8e92155959d9919114a0a2 in 950ms, sequenceid=412, compaction requested=true 2024-11-09T17:26:04,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:04,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:04,691 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:04,691 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:04,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:04,692 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:04,692 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:04,692 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:26:04,692 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:26:04,692 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,692 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,693 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3b4e1f15b180482390e8a9ea60031cde, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=39.3 K 2024-11-09T17:26:04,693 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/5983db997e5145c48ea92b933ab3e810, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.9 K 2024-11-09T17:26:04,693 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 5983db997e5145c48ea92b933ab3e810, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:04,693 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3b4e1f15b180482390e8a9ea60031cde, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:04,694 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 3bafe431968a4a1fadced3c049c97d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731173160954 2024-11-09T17:26:04,694 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7a7881753a38409396a49785de7f5bbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731173160954 2024-11-09T17:26:04,695 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 21583c56099e41bdabddd44dc7a72aa6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:04,695 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1fda7a4be84143509166b58ff0f92057, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:04,704 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:04,704 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a7ae7abe4b46462a8b022706d3b84da8 is 50, key is test_row_0/B:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:04,705 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:04,706 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/67614aa3b2c549c4add6950a5f021cee is 50, key is test_row_0/A:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:04,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741929_1105 (size=13323) 2024-11-09T17:26:04,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741930_1106 (size=13323) 2024-11-09T17:26:04,742 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/67614aa3b2c549c4add6950a5f021cee as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/67614aa3b2c549c4add6950a5f021cee 2024-11-09T17:26:04,751 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 67614aa3b2c549c4add6950a5f021cee(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:04,751 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:04,751 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173164690; duration=0sec 2024-11-09T17:26:04,752 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:04,752 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:26:04,752 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:04,753 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:04,753 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:26:04,753 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,753 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cba026849a3246988020b1c9ade3c63a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=36.9 K 2024-11-09T17:26:04,754 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cba026849a3246988020b1c9ade3c63a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731173160301 2024-11-09T17:26:04,754 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 35b07ce48f1e4c6096ae909d19b7ef2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=392, earliestPutTs=1731173160954 2024-11-09T17:26:04,755 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4d1ebe8029b0408fb26ae1325f40182c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:04,779 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#92 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:04,779 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/2322bae5bbca4087851b75780861e8e9 is 50, key is test_row_0/C:col10/1731173163126/Put/seqid=0 2024-11-09T17:26:04,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741931_1107 (size=13323) 2024-11-09T17:26:04,792 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:04,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=31 2024-11-09T17:26:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:04,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T17:26:04,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:04,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:04,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4e068e9b71164e27a2410761ca349bad is 50, key is test_row_0/A:col10/1731173163795/Put/seqid=0 2024-11-09T17:26:04,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741932_1108 (size=12301) 2024-11-09T17:26:04,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:04,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:04,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. as already flushing 2024-11-09T17:26:04,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173224936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 328 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173225038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:05,130 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/a7ae7abe4b46462a8b022706d3b84da8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7ae7abe4b46462a8b022706d3b84da8 2024-11-09T17:26:05,136 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into a7ae7abe4b46462a8b022706d3b84da8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:05,136 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:05,136 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173164691; duration=0sec 2024-11-09T17:26:05,136 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:05,136 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:26:05,189 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/2322bae5bbca4087851b75780861e8e9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2322bae5bbca4087851b75780861e8e9 2024-11-09T17:26:05,194 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 2322bae5bbca4087851b75780861e8e9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:05,194 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:05,194 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173164691; duration=0sec 2024-11-09T17:26:05,195 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:05,195 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:26:05,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4e068e9b71164e27a2410761ca349bad 2024-11-09T17:26:05,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/ece91879264145e59e53e5edb428c034 is 50, key is test_row_0/B:col10/1731173163795/Put/seqid=0 2024-11-09T17:26:05,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741933_1109 (size=12301) 2024-11-09T17:26:05,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173225240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:05,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 332 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173225546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:05,617 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/ece91879264145e59e53e5edb428c034 2024-11-09T17:26:05,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/87e2999ab457445786979068869284c4 is 50, key is test_row_0/C:col10/1731173163795/Put/seqid=0 2024-11-09T17:26:05,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741934_1110 (size=12301) 2024-11-09T17:26:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:06,030 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/87e2999ab457445786979068869284c4 2024-11-09T17:26:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4e068e9b71164e27a2410761ca349bad as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad 2024-11-09T17:26:06,040 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad, entries=150, sequenceid=432, filesize=12.0 K 2024-11-09T17:26:06,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/ece91879264145e59e53e5edb428c034 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034 2024-11-09T17:26:06,046 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034, entries=150, sequenceid=432, filesize=12.0 K 2024-11-09T17:26:06,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/87e2999ab457445786979068869284c4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4 2024-11-09T17:26:06,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:06,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:46744 deadline: 1731173226049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:06,052 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4, entries=150, sequenceid=432, filesize=12.0 K 2024-11-09T17:26:06,053 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for d0b59f7cee8e92155959d9919114a0a2 in 1260ms, sequenceid=432, compaction requested=false 2024-11-09T17:26:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:06,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=31}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=31 2024-11-09T17:26:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=31 2024-11-09T17:26:06,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=31, resume processing ppid=30 2024-11-09T17:26:06,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, ppid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3310 sec 2024-11-09T17:26:06,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=30, table=TestAcidGuarantees in 2.3340 sec 2024-11-09T17:26:06,507 DEBUG [Thread-160 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x295cb1ac to 127.0.0.1:57222 2024-11-09T17:26:06,507 DEBUG [Thread-166 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c915d17 to 127.0.0.1:57222 2024-11-09T17:26:06,507 DEBUG [Thread-166 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:06,507 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:06,507 DEBUG [Thread-162 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70267494 to 127.0.0.1:57222 2024-11-09T17:26:06,507 DEBUG [Thread-162 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:06,508 DEBUG [Thread-164 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d2a8e08 to 127.0.0.1:57222 2024-11-09T17:26:06,508 DEBUG [Thread-164 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:06,580 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:26:06,762 ERROR [LeaseRenewer:jenkins@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:07,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:07,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T17:26:07,079 DEBUG [Thread-149 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f343a4d to 127.0.0.1:57222 2024-11-09T17:26:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:07,079 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:07,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:07,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:07,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4c5870c6a2b94719aa3f9dd3e299b68e is 50, key is test_row_0/A:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:07,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741935_1111 (size=12301) 2024-11-09T17:26:07,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4c5870c6a2b94719aa3f9dd3e299b68e 2024-11-09T17:26:07,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fed6c637e88d4eaa9ebdaa5a27468f0a is 50, key is test_row_0/B:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:07,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741936_1112 (size=12301) 2024-11-09T17:26:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=30 2024-11-09T17:26:07,834 INFO [Thread-159 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 30 completed 2024-11-09T17:26:07,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fed6c637e88d4eaa9ebdaa5a27468f0a 2024-11-09T17:26:07,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fffd58f3fd58465bb3b1f1c585173e49 is 50, key is test_row_0/C:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:07,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741937_1113 (size=12301) 2024-11-09T17:26:08,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fffd58f3fd58465bb3b1f1c585173e49 2024-11-09T17:26:08,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/4c5870c6a2b94719aa3f9dd3e299b68e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e 2024-11-09T17:26:08,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e, entries=150, sequenceid=452, filesize=12.0 K 2024-11-09T17:26:08,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fed6c637e88d4eaa9ebdaa5a27468f0a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a 2024-11-09T17:26:08,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a, entries=150, sequenceid=452, filesize=12.0 K 2024-11-09T17:26:08,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/fffd58f3fd58465bb3b1f1c585173e49 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49 2024-11-09T17:26:08,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49, entries=150, sequenceid=452, filesize=12.0 K 2024-11-09T17:26:08,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=0 B/0 for d0b59f7cee8e92155959d9919114a0a2 in 1277ms, sequenceid=452, compaction requested=true 2024-11-09T17:26:08,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:08,357 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:08,357 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0b59f7cee8e92155959d9919114a0a2:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:08,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/B is initiating minor compaction (all files) 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/A is initiating minor compaction (all files) 2024-11-09T17:26:08,358 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/B in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:08,358 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/A in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:08,358 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/67614aa3b2c549c4add6950a5f021cee, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=37.0 K 2024-11-09T17:26:08,358 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7ae7abe4b46462a8b022706d3b84da8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=37.0 K 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 67614aa3b2c549c4add6950a5f021cee, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:08,358 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a7ae7abe4b46462a8b022706d3b84da8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:08,359 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4e068e9b71164e27a2410761ca349bad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731173163783 2024-11-09T17:26:08,359 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ece91879264145e59e53e5edb428c034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731173163783 2024-11-09T17:26:08,359 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4c5870c6a2b94719aa3f9dd3e299b68e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731173164925 2024-11-09T17:26:08,359 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting fed6c637e88d4eaa9ebdaa5a27468f0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731173164925 2024-11-09T17:26:08,366 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:08,367 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:08,367 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1edc2e0778074bc2b9eca8489ad343b2 is 50, key is test_row_0/A:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:08,367 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/11a8819659ea414b84581faeb2e738fa is 50, key is test_row_0/B:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:08,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741938_1114 (size=13425) 2024-11-09T17:26:08,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741939_1115 (size=13425) 2024-11-09T17:26:08,785 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/11a8819659ea414b84581faeb2e738fa as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/11a8819659ea414b84581faeb2e738fa 2024-11-09T17:26:08,785 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1edc2e0778074bc2b9eca8489ad343b2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1edc2e0778074bc2b9eca8489ad343b2 2024-11-09T17:26:08,791 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/B of d0b59f7cee8e92155959d9919114a0a2 into 11a8819659ea414b84581faeb2e738fa(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:08,791 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/A of d0b59f7cee8e92155959d9919114a0a2 into 1edc2e0778074bc2b9eca8489ad343b2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:08,791 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/B, priority=13, startTime=1731173168357; duration=0sec 2024-11-09T17:26:08,791 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/A, priority=13, startTime=1731173168356; duration=0sec 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:B 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:A 2024-11-09T17:26:08,791 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:08,792 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:08,792 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d0b59f7cee8e92155959d9919114a0a2/C is initiating minor compaction (all files) 2024-11-09T17:26:08,792 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d0b59f7cee8e92155959d9919114a0a2/C in TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:08,793 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2322bae5bbca4087851b75780861e8e9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp, totalSize=37.0 K 2024-11-09T17:26:08,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2322bae5bbca4087851b75780861e8e9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731173163122 2024-11-09T17:26:08,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 87e2999ab457445786979068869284c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731173163783 2024-11-09T17:26:08,794 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting fffd58f3fd58465bb3b1f1c585173e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1731173164925 2024-11-09T17:26:08,801 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0b59f7cee8e92155959d9919114a0a2#C#compaction#101 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:08,801 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/768ab6a2dbbf4276a7c53596555214d6 is 50, key is test_row_0/C:col10/1731173167062/Put/seqid=0 2024-11-09T17:26:08,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741940_1116 (size=13425) 2024-11-09T17:26:09,221 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/768ab6a2dbbf4276a7c53596555214d6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/768ab6a2dbbf4276a7c53596555214d6 2024-11-09T17:26:09,226 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d0b59f7cee8e92155959d9919114a0a2/C of d0b59f7cee8e92155959d9919114a0a2 into 768ab6a2dbbf4276a7c53596555214d6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:09,227 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:09,227 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2., storeName=d0b59f7cee8e92155959d9919114a0a2/C, priority=13, startTime=1731173168357; duration=0sec 2024-11-09T17:26:09,227 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:09,227 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0b59f7cee8e92155959d9919114a0a2:C 2024-11-09T17:26:11,118 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:11,558 DEBUG [Thread-151 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22cb07dd to 127.0.0.1:57222 2024-11-09T17:26:11,558 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:11,566 DEBUG [Thread-153 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x478bae6b to 127.0.0.1:57222 2024-11-09T17:26:11,566 DEBUG [Thread-153 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:11,575 DEBUG [Thread-157 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38766d64 to 127.0.0.1:57222 2024-11-09T17:26:11,575 DEBUG [Thread-157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:11,595 DEBUG [Thread-155 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5400112e to 127.0.0.1:57222 2024-11-09T17:26:11,595 DEBUG [Thread-155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 216 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6307 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6102 2024-11-09T17:26:11,595 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:26:11,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2661 2024-11-09T17:26:11,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7976 rows 2024-11-09T17:26:11,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2639 2024-11-09T17:26:11,596 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7913 rows 2024-11-09T17:26:11,596 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:26:11,596 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f6e36fe to 127.0.0.1:57222 2024-11-09T17:26:11,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:11,598 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:26:11,602 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:26:11,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=32, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:11,609 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173171608"}]},"ts":"1731173171608"} 2024-11-09T17:26:11,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:11,610 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:26:11,669 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:26:11,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:26:11,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, UNASSIGN}] 2024-11-09T17:26:11,676 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=34, ppid=33, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, UNASSIGN 2024-11-09T17:26:11,677 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=d0b59f7cee8e92155959d9919114a0a2, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:11,678 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:26:11,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; CloseRegionProcedure d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:11,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:11,817 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T17:26:11,820 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55994, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T17:26:11,834 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:11,835 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] handler.UnassignRegionHandler(124): Close d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:11,835 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:26:11,836 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1681): Closing d0b59f7cee8e92155959d9919114a0a2, disabling compactions & flushes 2024-11-09T17:26:11,836 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:11,836 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:11,836 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. after waiting 0 ms 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:11,837 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(2837): Flushing d0b59f7cee8e92155959d9919114a0a2 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=A 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=B 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:11,837 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0b59f7cee8e92155959d9919114a0a2, store=C 2024-11-09T17:26:11,838 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:11,845 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1b1de0b330dd4ee8bd6f08f7af84cfff is 50, key is test_row_0/A:col10/1731173171594/Put/seqid=0 2024-11-09T17:26:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741941_1117 (size=9857) 2024-11-09T17:26:11,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:12,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:12,252 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1b1de0b330dd4ee8bd6f08f7af84cfff 2024-11-09T17:26:12,266 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc7d046443b94b09ab0bcc02c5a2f6f3 is 50, key is test_row_0/B:col10/1731173171594/Put/seqid=0 2024-11-09T17:26:12,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741942_1118 (size=9857) 2024-11-09T17:26:12,673 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc7d046443b94b09ab0bcc02c5a2f6f3 2024-11-09T17:26:12,686 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/492ac5f055be4a169ab3285839b5bf1e is 50, key is test_row_0/C:col10/1731173171594/Put/seqid=0 2024-11-09T17:26:12,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741943_1119 (size=9857) 2024-11-09T17:26:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:13,092 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/492ac5f055be4a169ab3285839b5bf1e 2024-11-09T17:26:13,104 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/A/1b1de0b330dd4ee8bd6f08f7af84cfff as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1b1de0b330dd4ee8bd6f08f7af84cfff 2024-11-09T17:26:13,110 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1b1de0b330dd4ee8bd6f08f7af84cfff, entries=100, sequenceid=462, filesize=9.6 K 2024-11-09T17:26:13,111 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/B/fc7d046443b94b09ab0bcc02c5a2f6f3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc7d046443b94b09ab0bcc02c5a2f6f3 2024-11-09T17:26:13,116 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc7d046443b94b09ab0bcc02c5a2f6f3, entries=100, sequenceid=462, filesize=9.6 K 2024-11-09T17:26:13,117 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/.tmp/C/492ac5f055be4a169ab3285839b5bf1e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/492ac5f055be4a169ab3285839b5bf1e 2024-11-09T17:26:13,120 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/492ac5f055be4a169ab3285839b5bf1e, entries=100, sequenceid=462, filesize=9.6 K 2024-11-09T17:26:13,121 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d0b59f7cee8e92155959d9919114a0a2 in 1284ms, sequenceid=462, compaction requested=false 2024-11-09T17:26:13,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3c48ea2dfa724a2d8139aea597fb723d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3927dcd5e6584f93bc5642c7e53758a9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b47a759339464ba588216a53bd03c4a0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/be56424edf5343fc8bbcf17ec2822291, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9d947a8336814b76bf3934186856e399, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/832843c92c6a4625a20b7ce4bb2f4d18, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3758be1755a5489abb6bb2137d7397a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/40074587e9c94e2abb6e9f2073976c42, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3b4e1f15b180482390e8a9ea60031cde, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/67614aa3b2c549c4add6950a5f021cee, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e] to archive 2024-11-09T17:26:13,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:13,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4385ce6a9f334720a51d1642fe63f76d 2024-11-09T17:26:13,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/276b1c4ad1584194b19a36cf59c3ebb2 2024-11-09T17:26:13,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/66458c15bd1d48b994068f175bcbff27 2024-11-09T17:26:13,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3c48ea2dfa724a2d8139aea597fb723d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3c48ea2dfa724a2d8139aea597fb723d 2024-11-09T17:26:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/37ccf8bb3a9e4d4fb25469bc7643fb0b 2024-11-09T17:26:13,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3927dcd5e6584f93bc5642c7e53758a9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3927dcd5e6584f93bc5642c7e53758a9 2024-11-09T17:26:13,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/54c03629fa6d467a995b424c9ca8de53 2024-11-09T17:26:13,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1855c3a3f68a4137bff61cb4fd4427c1 2024-11-09T17:26:13,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e8c5c32863d4f7b842548fbed2d83b6 2024-11-09T17:26:13,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b47a759339464ba588216a53bd03c4a0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b47a759339464ba588216a53bd03c4a0 2024-11-09T17:26:13,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/2e31e4d325d140679c469f99af4f5515 2024-11-09T17:26:13,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/be56424edf5343fc8bbcf17ec2822291 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/be56424edf5343fc8bbcf17ec2822291 2024-11-09T17:26:13,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/cf96207fadde462d8479c1ca6735ebb3 2024-11-09T17:26:13,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/f40cd4a790f54ee19eed8a11a47d9798 2024-11-09T17:26:13,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/63c375188beb4c6cadb02429d489fbb6 2024-11-09T17:26:13,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9d947a8336814b76bf3934186856e399 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9d947a8336814b76bf3934186856e399 2024-11-09T17:26:13,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b7990a79263c402aa28d40c67b5f3a22 2024-11-09T17:26:13,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/adcc607387be4eb2b63a4d7555e446c2 2024-11-09T17:26:13,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/832843c92c6a4625a20b7ce4bb2f4d18 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/832843c92c6a4625a20b7ce4bb2f4d18 2024-11-09T17:26:13,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/98b3a3ebda704a2bb406faa8bba34484 2024-11-09T17:26:13,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3758be1755a5489abb6bb2137d7397a7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3758be1755a5489abb6bb2137d7397a7 2024-11-09T17:26:13,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/bf182540ed3b4cd49540050ac8a70f49 2024-11-09T17:26:13,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/b18a28ee15944e3a9f93e71f749367c2 2024-11-09T17:26:13,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/81e44ffe46f547b69cc7c62235932ab0 2024-11-09T17:26:13,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/40074587e9c94e2abb6e9f2073976c42 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/40074587e9c94e2abb6e9f2073976c42 2024-11-09T17:26:13,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/750c29c88946469aa10be8ed1babe131 2024-11-09T17:26:13,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3b4e1f15b180482390e8a9ea60031cde to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/3b4e1f15b180482390e8a9ea60031cde 2024-11-09T17:26:13,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/9116c6aeb2fb44138ae06fb8dde13e66 2024-11-09T17:26:13,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/7a7881753a38409396a49785de7f5bbb 2024-11-09T17:26:13,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1fda7a4be84143509166b58ff0f92057 2024-11-09T17:26:13,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/67614aa3b2c549c4add6950a5f021cee to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/67614aa3b2c549c4add6950a5f021cee 2024-11-09T17:26:13,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4e068e9b71164e27a2410761ca349bad 2024-11-09T17:26:13,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/4c5870c6a2b94719aa3f9dd3e299b68e 2024-11-09T17:26:13,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b1d7b7a3ecb646fc9eae23079fe68b9c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/65a10973659248c28c5ae99074a6907b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/115a49c5cdbd4aa9b3caf659ae351f7c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/328745798efa4baf9ce909d5404fa86a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ce7feedf5144445088fe89f14ff6a7e5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8e3db8de376f473fb97bcbc7fdd1e9b1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/564ba6134d62454fa9d20d7ac6c9f7b5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/5983db997e5145c48ea92b933ab3e810, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7ae7abe4b46462a8b022706d3b84da8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a] to archive 2024-11-09T17:26:13,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:13,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/db016d2a30154160a45a958111c2ce04 2024-11-09T17:26:13,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/cb28b15ff01c423ba469ffd6eff365ce 2024-11-09T17:26:13,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b1d7b7a3ecb646fc9eae23079fe68b9c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b1d7b7a3ecb646fc9eae23079fe68b9c 2024-11-09T17:26:13,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/d111349409e2484a9ad2dcf76a2beef2 2024-11-09T17:26:13,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4ceed7bd6a6c4d3db08237423b355523 2024-11-09T17:26:13,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/65a10973659248c28c5ae99074a6907b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/65a10973659248c28c5ae99074a6907b 2024-11-09T17:26:13,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7830f241d8b42b0b850afbd926d8de4 2024-11-09T17:26:13,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/b74da3fd5888449586053afebece3093 2024-11-09T17:26:13,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/115a49c5cdbd4aa9b3caf659ae351f7c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/115a49c5cdbd4aa9b3caf659ae351f7c 2024-11-09T17:26:13,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/00032ee55e104164b78e41f672943d14 2024-11-09T17:26:13,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3b168af4410c41f894c01e5a09f517f7 2024-11-09T17:26:13,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/328745798efa4baf9ce909d5404fa86a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/328745798efa4baf9ce909d5404fa86a 2024-11-09T17:26:13,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/81ed6c5fb3724d6493b915d4424b0a14 2024-11-09T17:26:13,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/bc7161b0701d4c2e9631c0ed29893575 2024-11-09T17:26:13,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/6b2ee7daf22a4dc385f7b5bdc1fa8cbe 2024-11-09T17:26:13,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8528a7a2873b447e9ee0a42db62a4213 2024-11-09T17:26:13,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/7cad540176024fa59e0f9e2172c49978 2024-11-09T17:26:13,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ce7feedf5144445088fe89f14ff6a7e5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ce7feedf5144445088fe89f14ff6a7e5 2024-11-09T17:26:13,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/e32c516dd0134bd5b87b6ac354725cfc 2024-11-09T17:26:13,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/932e1ed5cf504b39b7f1797470545fec 2024-11-09T17:26:13,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8e3db8de376f473fb97bcbc7fdd1e9b1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/8e3db8de376f473fb97bcbc7fdd1e9b1 2024-11-09T17:26:13,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a5d6fb67084b4bc59fe4b1ff7c9f8ca0 2024-11-09T17:26:13,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/eeb83c9ed3b442cdb2bdc1a4ed29a186 2024-11-09T17:26:13,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/564ba6134d62454fa9d20d7ac6c9f7b5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/564ba6134d62454fa9d20d7ac6c9f7b5 2024-11-09T17:26:13,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/4f4f27e61bea42a7bf4522995078e2eb 2024-11-09T17:26:13,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/2cabee6f87bc4076a5a53f90252721a1 2024-11-09T17:26:13,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/5983db997e5145c48ea92b933ab3e810 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/5983db997e5145c48ea92b933ab3e810 2024-11-09T17:26:13,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc6286204fe34e21ac29f80cf2ae5b61 2024-11-09T17:26:13,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/3bafe431968a4a1fadced3c049c97d27 2024-11-09T17:26:13,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7ae7abe4b46462a8b022706d3b84da8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/a7ae7abe4b46462a8b022706d3b84da8 2024-11-09T17:26:13,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/21583c56099e41bdabddd44dc7a72aa6 2024-11-09T17:26:13,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/ece91879264145e59e53e5edb428c034 2024-11-09T17:26:13,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fed6c637e88d4eaa9ebdaa5a27468f0a 2024-11-09T17:26:13,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/bbb7b79b693f453b8d566caccb17c917, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/77003a63983044d3ae706268ac6c39d9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/e1a9f3a842c345029440870f21c55ff1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/a01757fac1314afeb70d5ae3697eb101, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/587bdd983b63440c8cd1c6180096cabd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/550712b8cafb409981baaecf618b560f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7cf32169c83f4fd4a36795e557dab3a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/eab74bc90e144d1b945eb64fa4c5dd90, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cba026849a3246988020b1c9ade3c63a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2322bae5bbca4087851b75780861e8e9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49] to archive 2024-11-09T17:26:13,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:13,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/41a9bfa19ba94713a3ad25a78c199d23 2024-11-09T17:26:13,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/01113e86195e4f1fbaa3b31ffa6ad223 2024-11-09T17:26:13,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/bbb7b79b693f453b8d566caccb17c917 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/bbb7b79b693f453b8d566caccb17c917 2024-11-09T17:26:13,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c4f55121e9f04a45a4267aefb11afdb0 2024-11-09T17:26:13,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/d758138668ea4fe1a1eae78087b76e56 2024-11-09T17:26:13,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/77003a63983044d3ae706268ac6c39d9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/77003a63983044d3ae706268ac6c39d9 2024-11-09T17:26:13,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ca878331f14b474c904019549a1d0915 2024-11-09T17:26:13,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fbe7dc89f5db457ca8485cd16ddcafb3 2024-11-09T17:26:13,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/e1a9f3a842c345029440870f21c55ff1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/e1a9f3a842c345029440870f21c55ff1 2024-11-09T17:26:13,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cd82d06ed39d4183859dbb8d4f243635 2024-11-09T17:26:13,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2c8f0e6cbf1745f7ba92a21f87614c8f 2024-11-09T17:26:13,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/a01757fac1314afeb70d5ae3697eb101 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/a01757fac1314afeb70d5ae3697eb101 2024-11-09T17:26:13,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/1ac3331961b34a65a4e926a5d46e0cc8 2024-11-09T17:26:13,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/c476e5a10a7047479ab6356db61ed8f7 2024-11-09T17:26:13,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/587bdd983b63440c8cd1c6180096cabd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/587bdd983b63440c8cd1c6180096cabd 2024-11-09T17:26:13,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/889220142fd5422ca1ede35fc85f1eda 2024-11-09T17:26:13,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8f6d8cd75d06441ba2da7b854a215731 2024-11-09T17:26:13,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/550712b8cafb409981baaecf618b560f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/550712b8cafb409981baaecf618b560f 2024-11-09T17:26:13,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ed7841695d0a4b2cb6c566ad7ae2d312 2024-11-09T17:26:13,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/69419ea7a4a845fe88a7c923d9e65009 2024-11-09T17:26:13,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7cf32169c83f4fd4a36795e557dab3a7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7cf32169c83f4fd4a36795e557dab3a7 2024-11-09T17:26:13,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/47c847d4da1b48cca7b1d125c56e4c4b 2024-11-09T17:26:13,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/0db0b51b75bf4b4caf4bbb654184cab8 2024-11-09T17:26:13,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/eab74bc90e144d1b945eb64fa4c5dd90 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/eab74bc90e144d1b945eb64fa4c5dd90 2024-11-09T17:26:13,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/ffa50c776e22452580227d19f583d300 2024-11-09T17:26:13,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/7053dcb514e54981b96d1579c7e7118c 2024-11-09T17:26:13,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cba026849a3246988020b1c9ade3c63a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/cba026849a3246988020b1c9ade3c63a 2024-11-09T17:26:13,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/8b79866593db45c897caa46f94e250da 2024-11-09T17:26:13,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/35b07ce48f1e4c6096ae909d19b7ef2e 2024-11-09T17:26:13,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2322bae5bbca4087851b75780861e8e9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/2322bae5bbca4087851b75780861e8e9 2024-11-09T17:26:13,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/4d1ebe8029b0408fb26ae1325f40182c 2024-11-09T17:26:13,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/87e2999ab457445786979068869284c4 2024-11-09T17:26:13,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/fffd58f3fd58465bb3b1f1c585173e49 2024-11-09T17:26:13,264 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/recovered.edits/465.seqid, newMaxSeqId=465, maxSeqId=1 2024-11-09T17:26:13,267 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2. 2024-11-09T17:26:13,267 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] regionserver.HRegion(1635): Region close journal for d0b59f7cee8e92155959d9919114a0a2: 2024-11-09T17:26:13,268 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=35}] handler.UnassignRegionHandler(170): Closed d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:13,269 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=34 updating hbase:meta row=d0b59f7cee8e92155959d9919114a0a2, regionState=CLOSED 2024-11-09T17:26:13,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-09T17:26:13,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; CloseRegionProcedure d0b59f7cee8e92155959d9919114a0a2, server=c28fd47d982f,45633,1731173138129 in 1.5920 sec 2024-11-09T17:26:13,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T17:26:13,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, ppid=33, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0b59f7cee8e92155959d9919114a0a2, UNASSIGN in 1.5960 sec 2024-11-09T17:26:13,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=33, resume processing ppid=32 2024-11-09T17:26:13,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=33, ppid=32, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6020 sec 2024-11-09T17:26:13,275 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173173275"}]},"ts":"1731173173275"} 2024-11-09T17:26:13,276 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:26:13,289 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:26:13,292 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.6860 sec 2024-11-09T17:26:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=32 2024-11-09T17:26:13,717 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 32 completed 2024-11-09T17:26:13,723 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:26:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,732 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=36, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,734 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=36, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T17:26:13,738 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:13,742 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/recovered.edits] 2024-11-09T17:26:13,745 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1b1de0b330dd4ee8bd6f08f7af84cfff to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1b1de0b330dd4ee8bd6f08f7af84cfff 2024-11-09T17:26:13,746 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1edc2e0778074bc2b9eca8489ad343b2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/A/1edc2e0778074bc2b9eca8489ad343b2 2024-11-09T17:26:13,748 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/11a8819659ea414b84581faeb2e738fa to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/11a8819659ea414b84581faeb2e738fa 2024-11-09T17:26:13,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc7d046443b94b09ab0bcc02c5a2f6f3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/B/fc7d046443b94b09ab0bcc02c5a2f6f3 2024-11-09T17:26:13,752 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/492ac5f055be4a169ab3285839b5bf1e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/492ac5f055be4a169ab3285839b5bf1e 2024-11-09T17:26:13,753 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/768ab6a2dbbf4276a7c53596555214d6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/C/768ab6a2dbbf4276a7c53596555214d6 2024-11-09T17:26:13,755 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/recovered.edits/465.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2/recovered.edits/465.seqid 2024-11-09T17:26:13,756 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d0b59f7cee8e92155959d9919114a0a2 2024-11-09T17:26:13,756 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:26:13,760 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=36, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T17:26:13,768 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:26:13,794 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:26:13,796 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=36, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,796 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:26:13,796 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173173796"}]},"ts":"9223372036854775807"} 2024-11-09T17:26:13,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:26:13,799 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d0b59f7cee8e92155959d9919114a0a2, NAME => 'TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:26:13,799 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:26:13,799 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173173799"}]},"ts":"9223372036854775807"} 2024-11-09T17:26:13,801 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:26:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T17:26:13,845 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=36, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:13,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=36, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 120 msec 2024-11-09T17:26:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-09T17:26:14,038 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 36 completed 2024-11-09T17:26:14,052 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMixedAtomicity Thread=237 (was 219) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;c28fd47d982f:45633-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-195685394_22 at /127.0.0.1:44192 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=456 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 174) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6066 (was 6592) 2024-11-09T17:26:14,063 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobMixedAtomicity Thread=237, OpenFileDescriptor=456, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=6066 2024-11-09T17:26:14,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:26:14,065 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:26:14,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:14,068 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:26:14,068 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:14,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 37 2024-11-09T17:26:14,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:14,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:26:14,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741944_1120 (size=960) 2024-11-09T17:26:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:14,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:14,483 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:26:14,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741945_1121 (size=53) 2024-11-09T17:26:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:14,895 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:26:14,895 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing 9b0fa22acef654074d43b8d7d6d68c6a, disabling compactions & flushes 2024-11-09T17:26:14,895 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:14,895 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:14,895 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. after waiting 0 ms 2024-11-09T17:26:14,895 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:14,895 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:14,896 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:14,898 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:26:14,899 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173174899"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173174899"}]},"ts":"1731173174899"} 2024-11-09T17:26:14,902 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:26:14,903 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:26:14,903 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173174903"}]},"ts":"1731173174903"} 2024-11-09T17:26:14,905 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:26:14,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, ASSIGN}] 2024-11-09T17:26:14,967 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=38, ppid=37, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, ASSIGN 2024-11-09T17:26:14,969 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=38, ppid=37, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:26:15,120 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=38 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:15,122 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE; OpenRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:15,277 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:15,285 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:15,285 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7285): Opening region: {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:26:15,286 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,286 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:26:15,286 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7327): checking encryption for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,286 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7330): checking classloading for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,289 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,291 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:15,291 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName A 2024-11-09T17:26:15,291 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:15,292 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:15,292 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,294 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:15,295 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName B 2024-11-09T17:26:15,295 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:15,296 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:15,296 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,298 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:15,298 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName C 2024-11-09T17:26:15,298 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:15,299 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:15,299 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:15,300 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,301 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,303 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:26:15,304 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1085): writing seq id for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:15,306 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:26:15,306 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1102): Opened 9b0fa22acef654074d43b8d7d6d68c6a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67318987, jitterRate=0.003131076693534851}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:26:15,307 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1001): Region open journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:15,308 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., pid=39, masterSystemTime=1731173175276 2024-11-09T17:26:15,309 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:15,309 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:15,310 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=38 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:15,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=39, resume processing ppid=38 2024-11-09T17:26:15,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=39, ppid=38, state=SUCCESS; OpenRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 in 189 msec 2024-11-09T17:26:15,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-09T17:26:15,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, ASSIGN in 348 msec 2024-11-09T17:26:15,315 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:26:15,316 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173175315"}]},"ts":"1731173175315"} 2024-11-09T17:26:15,317 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:26:15,362 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=37, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:26:15,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2970 sec 2024-11-09T17:26:16,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-09T17:26:16,179 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 37 completed 2024-11-09T17:26:16,184 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a9b9802 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@118b007e 2024-11-09T17:26:16,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29de25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:16,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:16,239 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33892, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:16,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:26:16,245 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56000, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:26:16,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:26:16,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:26:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:16,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741946_1122 (size=996) 2024-11-09T17:26:16,672 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-09T17:26:16,672 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-09T17:26:16,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:26:16,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, REOPEN/MOVE}] 2024-11-09T17:26:16,690 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, REOPEN/MOVE 2024-11-09T17:26:16,691 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:16,692 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:26:16,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE; CloseRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:16,843 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:16,845 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] handler.UnassignRegionHandler(124): Close 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:16,845 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:26:16,845 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1681): Closing 9b0fa22acef654074d43b8d7d6d68c6a, disabling compactions & flushes 2024-11-09T17:26:16,845 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:16,845 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:16,845 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. after waiting 0 ms 2024-11-09T17:26:16,846 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:16,856 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T17:26:16,858 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:16,858 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegion(1635): Region close journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:16,858 WARN [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] regionserver.HRegionServer(3786): Not adding moved region record: 9b0fa22acef654074d43b8d7d6d68c6a to self. 2024-11-09T17:26:16,860 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=43}] handler.UnassignRegionHandler(170): Closed 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:16,861 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=CLOSED 2024-11-09T17:26:16,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T17:26:16,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=43, ppid=42, state=SUCCESS; CloseRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 in 171 msec 2024-11-09T17:26:16,866 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=42, ppid=41, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, REOPEN/MOVE; state=CLOSED, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=true 2024-11-09T17:26:17,016 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=44, ppid=42, state=RUNNABLE; OpenRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:17,170 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,179 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,179 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7285): Opening region: {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:26:17,179 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,180 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:26:17,180 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7327): checking encryption for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,180 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7330): checking classloading for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,183 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,184 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:17,189 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName A 2024-11-09T17:26:17,192 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:17,193 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:17,193 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,194 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:17,194 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName B 2024-11-09T17:26:17,194 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:17,195 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:17,195 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,196 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:17,196 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b0fa22acef654074d43b8d7d6d68c6a columnFamilyName C 2024-11-09T17:26:17,196 DEBUG [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:17,196 INFO [StoreOpener-9b0fa22acef654074d43b8d7d6d68c6a-1 {}] regionserver.HStore(327): Store=9b0fa22acef654074d43b8d7d6d68c6a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:17,196 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,197 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,198 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,199 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:26:17,201 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1085): writing seq id for 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,201 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1102): Opened 9b0fa22acef654074d43b8d7d6d68c6a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69562179, jitterRate=0.03655724227428436}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:26:17,202 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1001): Region open journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:17,203 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., pid=44, masterSystemTime=1731173177170 2024-11-09T17:26:17,204 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,205 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,205 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=42 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=OPEN, openSeqNum=5, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=44, resume processing ppid=42 2024-11-09T17:26:17,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=44, ppid=42, state=SUCCESS; OpenRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 in 188 msec 2024-11-09T17:26:17,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-11-09T17:26:17,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, REOPEN/MOVE in 518 msec 2024-11-09T17:26:17,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T17:26:17,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, ppid=40, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-09T17:26:17,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=40, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 959 msec 2024-11-09T17:26:17,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-11-09T17:26:17,220 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7cae6c5c to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79982672 2024-11-09T17:26:17,283 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@433e2b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,285 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c820ef9 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b4bd1ba 2024-11-09T17:26:17,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176c5c1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,302 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b44b1e5 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@454f1431 2024-11-09T17:26:17,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190853fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,320 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42e904d8 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@505d5ccd 2024-11-09T17:26:17,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c5c4716, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,333 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a4c53ed to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@367f47f7 2024-11-09T17:26:17,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885d2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,342 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x247c0c93 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@22e911df 2024-11-09T17:26:17,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78cafade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,355 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x517ff977 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b727d6e 2024-11-09T17:26:17,365 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14c16cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,368 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3448d233 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c7940d9 2024-11-09T17:26:17,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341384e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,383 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a11164b to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c38ee58 2024-11-09T17:26:17,396 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26b120d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:17,402 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:17,402 DEBUG [hconnection-0x5b607ff4-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,403 DEBUG [hconnection-0x55c3144e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees 2024-11-09T17:26:17,403 DEBUG [hconnection-0x2c47e95c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,404 DEBUG [hconnection-0x2a49bace-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,404 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=45, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:17,405 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33904, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,405 DEBUG [hconnection-0x615bfe4c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,405 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33906, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:17,405 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33938, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,405 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=45, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:17,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:17,406 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33942, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,406 DEBUG [hconnection-0x5b774629-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,407 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33948, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,411 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33934, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,411 DEBUG [hconnection-0x761d908d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,414 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33962, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:26:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:17,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:17,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:17,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:17,419 DEBUG [hconnection-0x343da13f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,420 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33978, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,420 DEBUG [hconnection-0x35e09396-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:17,421 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33982, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:17,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173237452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173237454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173237457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173237456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173237458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094030f53fc8dc4a4e8f87783523aed765_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173177416/Put/seqid=0 2024-11-09T17:26:17,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741947_1123 (size=12154) 2024-11-09T17:26:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:17,558 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173237561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173237562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173237562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173237562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173237563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:17,713 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173237765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173237765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173237765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173237765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:17,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173237769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,866 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:17,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:17,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:17,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:17,896 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094030f53fc8dc4a4e8f87783523aed765_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094030f53fc8dc4a4e8f87783523aed765_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:17,897 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/cf3be8e90a26435399110dc944a29455, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:17,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/cf3be8e90a26435399110dc944a29455 is 175, key is test_row_0/A:col10/1731173177416/Put/seqid=0 2024-11-09T17:26:17,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741948_1124 (size=30955) 2024-11-09T17:26:17,909 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/cf3be8e90a26435399110dc944a29455 2024-11-09T17:26:17,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/3f7e5443a009496994bc7b5387dba600 is 50, key is test_row_0/B:col10/1731173177416/Put/seqid=0 2024-11-09T17:26:17,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741949_1125 (size=12001) 2024-11-09T17:26:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:18,019 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173238067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173238068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173238070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173238070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173238073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,172 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,326 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] handler.RSProcedureHandler(58): pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=46 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=46 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:18,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/3f7e5443a009496994bc7b5387dba600 2024-11-09T17:26:18,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44b42c31cc4942b68759b15ee588ba7b is 50, key is test_row_0/C:col10/1731173177416/Put/seqid=0 2024-11-09T17:26:18,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741950_1126 (size=12001) 2024-11-09T17:26:18,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44b42c31cc4942b68759b15ee588ba7b 2024-11-09T17:26:18,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/cf3be8e90a26435399110dc944a29455 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455 2024-11-09T17:26:18,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455, entries=150, sequenceid=17, filesize=30.2 K 2024-11-09T17:26:18,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/3f7e5443a009496994bc7b5387dba600 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600 2024-11-09T17:26:18,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:26:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44b42c31cc4942b68759b15ee588ba7b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b 2024-11-09T17:26:18,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:26:18,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 9b0fa22acef654074d43b8d7d6d68c6a in 1034ms, sequenceid=17, compaction requested=false 2024-11-09T17:26:18,453 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T17:26:18,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:18,479 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=46 2024-11-09T17:26:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:18,480 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T17:26:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:18,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:18,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:18,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110913cd058087344fe1a2bfc5dedd85d513_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173177453/Put/seqid=0 2024-11-09T17:26:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:18,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741951_1127 (size=12154) 2024-11-09T17:26:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:18,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:18,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173238583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173238587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173238585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173238584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173238588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173238694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173238694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173238694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173238694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173238694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,797 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:26:18,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173238896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173238898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173238898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173238899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173238899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:18,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:18,925 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110913cd058087344fe1a2bfc5dedd85d513_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110913cd058087344fe1a2bfc5dedd85d513_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f62386d323c6402daf88bc6aa87f34e3, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:18,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f62386d323c6402daf88bc6aa87f34e3 is 175, key is test_row_0/A:col10/1731173177453/Put/seqid=0 2024-11-09T17:26:18,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741952_1128 (size=30955) 2024-11-09T17:26:18,941 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f62386d323c6402daf88bc6aa87f34e3 2024-11-09T17:26:18,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/2000af22c567471e8f9686c535abab5f is 50, key is test_row_0/B:col10/1731173177453/Put/seqid=0 2024-11-09T17:26:18,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741953_1129 (size=12001) 2024-11-09T17:26:18,966 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/2000af22c567471e8f9686c535abab5f 2024-11-09T17:26:18,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f5fb89252c894114aab8e5795f66fbd2 is 50, key is test_row_0/C:col10/1731173177453/Put/seqid=0 2024-11-09T17:26:18,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741954_1130 (size=12001) 2024-11-09T17:26:18,992 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f5fb89252c894114aab8e5795f66fbd2 2024-11-09T17:26:18,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f62386d323c6402daf88bc6aa87f34e3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3 2024-11-09T17:26:19,005 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T17:26:19,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/2000af22c567471e8f9686c535abab5f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f 2024-11-09T17:26:19,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T17:26:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f5fb89252c894114aab8e5795f66fbd2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2 2024-11-09T17:26:19,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T17:26:19,020 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 9b0fa22acef654074d43b8d7d6d68c6a in 540ms, sequenceid=41, compaction requested=false 2024-11-09T17:26:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-11-09T17:26:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-11-09T17:26:19,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-09T17:26:19,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6170 sec 2024-11-09T17:26:19,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=45, table=TestAcidGuarantees in 1.6230 sec 2024-11-09T17:26:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:19,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T17:26:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:19,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:19,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ba1cd9808c0a4f66a0289088aae6969e_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:19,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173239229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173239229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173239231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173239231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173239233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741955_1131 (size=14594) 2024-11-09T17:26:19,258 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:19,263 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ba1cd9808c0a4f66a0289088aae6969e_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba1cd9808c0a4f66a0289088aae6969e_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:19,264 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/a67854eaebdb496bb91778c8a97f0abb, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:19,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/a67854eaebdb496bb91778c8a97f0abb is 175, key is test_row_0/A:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:19,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741956_1132 (size=39549) 2024-11-09T17:26:19,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/a67854eaebdb496bb91778c8a97f0abb 2024-11-09T17:26:19,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/aba1431ae1834ce184b83b92a309e9db is 50, key is test_row_0/B:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:19,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741957_1133 (size=12001) 2024-11-09T17:26:19,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173239335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173239336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173239337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173239337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173239338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-09T17:26:19,510 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 45 completed 2024-11-09T17:26:19,513 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees 2024-11-09T17:26:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:19,515 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=47, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:19,516 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=47, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:19,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173239538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173239539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173239543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173239544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173239545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:19,668 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-09T17:26:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/aba1431ae1834ce184b83b92a309e9db 2024-11-09T17:26:19,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/6ec7bdbd4ea140dcbf934b1a81f70a28 is 50, key is test_row_0/C:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:19,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741958_1134 (size=12001) 2024-11-09T17:26:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:19,821 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-09T17:26:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173239843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173239844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173239847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173239847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173239851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,974 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:19,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-09T17:26:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:19,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:19,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:20,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:20,127 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-09T17:26:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] handler.RSProcedureHandler(58): pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:20,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=48 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:20,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=48 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:20,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/6ec7bdbd4ea140dcbf934b1a81f70a28 2024-11-09T17:26:20,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/a67854eaebdb496bb91778c8a97f0abb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb 2024-11-09T17:26:20,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb, entries=200, sequenceid=54, filesize=38.6 K 2024-11-09T17:26:20,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/aba1431ae1834ce184b83b92a309e9db as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db 2024-11-09T17:26:20,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db, entries=150, sequenceid=54, filesize=11.7 K 2024-11-09T17:26:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/6ec7bdbd4ea140dcbf934b1a81f70a28 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28 2024-11-09T17:26:20,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28, entries=150, sequenceid=54, filesize=11.7 K 2024-11-09T17:26:20,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 9b0fa22acef654074d43b8d7d6d68c6a in 974ms, sequenceid=54, compaction requested=true 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:20,175 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:20,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:20,175 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:20,176 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:20,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:20,176 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:20,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:20,176 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,176 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,177 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=99.1 K 2024-11-09T17:26:20,177 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=35.2 K 2024-11-09T17:26:20,177 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,177 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb] 2024-11-09T17:26:20,177 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 3f7e5443a009496994bc7b5387dba600, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173177411 2024-11-09T17:26:20,177 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cf3be8e90a26435399110dc944a29455, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173177411 2024-11-09T17:26:20,177 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2000af22c567471e8f9686c535abab5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173177434 2024-11-09T17:26:20,178 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f62386d323c6402daf88bc6aa87f34e3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173177434 2024-11-09T17:26:20,178 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting aba1431ae1834ce184b83b92a309e9db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178585 2024-11-09T17:26:20,178 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting a67854eaebdb496bb91778c8a97f0abb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178584 2024-11-09T17:26:20,193 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:20,193 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/bbe92a3e07c44b6a9ba791fb78a42b61 is 50, key is test_row_0/B:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:20,195 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:20,203 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ac7166c8622746669eab185e9c3a8a89_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:20,211 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ac7166c8622746669eab185e9c3a8a89_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:20,211 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ac7166c8622746669eab185e9c3a8a89_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:20,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741960_1136 (size=4469) 2024-11-09T17:26:20,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741959_1135 (size=12104) 2024-11-09T17:26:20,245 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#115 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:20,247 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/53ceb31afd88431abbcfc18c21743300 is 175, key is test_row_0/A:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:20,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741961_1137 (size=31058) 2024-11-09T17:26:20,254 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/bbe92a3e07c44b6a9ba791fb78a42b61 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bbe92a3e07c44b6a9ba791fb78a42b61 2024-11-09T17:26:20,259 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/53ceb31afd88431abbcfc18c21743300 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300 2024-11-09T17:26:20,263 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into bbe92a3e07c44b6a9ba791fb78a42b61(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:20,263 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:20,263 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=13, startTime=1731173180175; duration=0sec 2024-11-09T17:26:20,263 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:20,263 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:20,263 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:20,266 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:20,266 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:20,266 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,266 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=35.2 K 2024-11-09T17:26:20,267 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 44b42c31cc4942b68759b15ee588ba7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173177411 2024-11-09T17:26:20,268 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f5fb89252c894114aab8e5795f66fbd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173177434 2024-11-09T17:26:20,268 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 6ec7bdbd4ea140dcbf934b1a81f70a28, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178585 2024-11-09T17:26:20,270 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into 53ceb31afd88431abbcfc18c21743300(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:20,271 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:20,271 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=13, startTime=1731173180175; duration=0sec 2024-11-09T17:26:20,271 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:20,271 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:20,279 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#116 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:20,279 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/ee8e802969fd4afb8bc9522901e048e3 is 50, key is test_row_0/C:col10/1731173178585/Put/seqid=0 2024-11-09T17:26:20,280 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=48 2024-11-09T17:26:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:26:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:20,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:20,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:20,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110937b21b021b064bebabfab86873185c84_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:20,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741962_1138 (size=12104) 2024-11-09T17:26:20,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741963_1139 (size=12154) 2024-11-09T17:26:20,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:20,339 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110937b21b021b064bebabfab86873185c84_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110937b21b021b064bebabfab86873185c84_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f011e8970a2249f39bfcfcc19def0624, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:20,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f011e8970a2249f39bfcfcc19def0624 is 175, key is test_row_0/A:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741964_1140 (size=30955) 2024-11-09T17:26:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:20,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:20,351 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f011e8970a2249f39bfcfcc19def0624 2024-11-09T17:26:20,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e8d271f9412c42b0a0388a2caedee08a is 50, key is test_row_0/B:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:20,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173240361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173240362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173240364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173240367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173240371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741965_1141 (size=12001) 2024-11-09T17:26:20,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173240465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173240468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173240468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173240469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173240473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:20,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173240668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173240671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173240672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173240673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173240676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,717 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/ee8e802969fd4afb8bc9522901e048e3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/ee8e802969fd4afb8bc9522901e048e3 2024-11-09T17:26:20,723 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into ee8e802969fd4afb8bc9522901e048e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:20,723 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:20,723 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=13, startTime=1731173180175; duration=0sec 2024-11-09T17:26:20,723 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:20,723 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:20,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e8d271f9412c42b0a0388a2caedee08a 2024-11-09T17:26:20,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e6e66adc88db486fb0ebfa237d1a3224 is 50, key is test_row_0/C:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741966_1142 (size=12001) 2024-11-09T17:26:20,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173240972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173240974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173240975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173240978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:20,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:20,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173240979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e6e66adc88db486fb0ebfa237d1a3224 2024-11-09T17:26:21,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/f011e8970a2249f39bfcfcc19def0624 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624 2024-11-09T17:26:21,198 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624, entries=150, sequenceid=79, filesize=30.2 K 2024-11-09T17:26:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e8d271f9412c42b0a0388a2caedee08a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a 2024-11-09T17:26:21,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T17:26:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e6e66adc88db486fb0ebfa237d1a3224 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224 2024-11-09T17:26:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,210 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T17:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9b0fa22acef654074d43b8d7d6d68c6a in 931ms, sequenceid=79, compaction requested=false 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:21,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-11-09T17:26:21,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-11-09T17:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=47, table=TestAcidGuarantees in 1.7030 sec 2024-11-09T17:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:26:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:21,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109571ad924dc6b4a7dbc5a992aa3e959a9_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_1/A:col10/1731173180362/Put/seqid=0 2024-11-09T17:26:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741967_1143 (size=12154) 2024-11-09T17:26:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173241520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,533 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109571ad924dc6b4a7dbc5a992aa3e959a9_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109571ad924dc6b4a7dbc5a992aa3e959a9_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,534 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/d67b6ea1f1de4d9eb7425013a0d49ffd, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:21,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173241523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/d67b6ea1f1de4d9eb7425013a0d49ffd is 175, key is test_row_1/A:col10/1731173180362/Put/seqid=0 2024-11-09T17:26:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173241524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173241526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173241528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741968_1144 (size=30951) 2024-11-09T17:26:21,555 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/d67b6ea1f1de4d9eb7425013a0d49ffd 2024-11-09T17:26:21,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f4cdeb355ec54fa5919fe0e0cc830951 is 50, key is test_row_1/B:col10/1731173180362/Put/seqid=0 2024-11-09T17:26:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741969_1145 (size=9657) 2024-11-09T17:26:21,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f4cdeb355ec54fa5919fe0e0cc830951 2024-11-09T17:26:21,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/2efbff6657b94c688b75e1da7824921d is 50, key is test_row_1/C:col10/1731173180362/Put/seqid=0 2024-11-09T17:26:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-09T17:26:21,620 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 47 completed 2024-11-09T17:26:21,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741970_1146 (size=9657) 2024-11-09T17:26:21,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees 2024-11-09T17:26:21,623 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:21,624 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=49, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:21,624 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173241630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173241635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173241636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173241637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173241639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:21,777 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-09T17:26:21,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:21,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:21,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173241833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173241839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173241840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173241840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173241841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:21,930 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:21,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-09T17:26:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:21,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] handler.RSProcedureHandler(58): pid=50 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=50 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:21,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=50 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:22,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/2efbff6657b94c688b75e1da7824921d 2024-11-09T17:26:22,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/d67b6ea1f1de4d9eb7425013a0d49ffd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd 2024-11-09T17:26:22,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd, entries=150, sequenceid=94, filesize=30.2 K 2024-11-09T17:26:22,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f4cdeb355ec54fa5919fe0e0cc830951 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951 2024-11-09T17:26:22,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951, entries=100, sequenceid=94, filesize=9.4 K 2024-11-09T17:26:22,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/2efbff6657b94c688b75e1da7824921d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d 2024-11-09T17:26:22,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d, entries=100, sequenceid=94, filesize=9.4 K 2024-11-09T17:26:22,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9b0fa22acef654074d43b8d7d6d68c6a in 551ms, sequenceid=94, compaction requested=true 2024-11-09T17:26:22,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:22,044 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:22,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:22,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:22,044 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:22,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:22,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:22,045 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:22,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:22,045 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:22,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:22,045 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:22,045 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:22,046 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=90.8 K 2024-11-09T17:26:22,046 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bbe92a3e07c44b6a9ba791fb78a42b61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=33.0 K 2024-11-09T17:26:22,046 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:22,046 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd] 2024-11-09T17:26:22,046 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting bbe92a3e07c44b6a9ba791fb78a42b61, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178585 2024-11-09T17:26:22,046 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 53ceb31afd88431abbcfc18c21743300, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178585 2024-11-09T17:26:22,046 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e8d271f9412c42b0a0388a2caedee08a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173179224 2024-11-09T17:26:22,047 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f011e8970a2249f39bfcfcc19def0624, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173179224 2024-11-09T17:26:22,047 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f4cdeb355ec54fa5919fe0e0cc830951, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173180362 2024-11-09T17:26:22,047 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d67b6ea1f1de4d9eb7425013a0d49ffd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173180360 2024-11-09T17:26:22,057 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:22,058 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/8e83e3cd7a9e48c182360dba27a3a433 is 50, key is test_row_0/B:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:22,061 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:22,075 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109507615ce716f403fb0302fcf542a351b_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:22,077 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109507615ce716f403fb0302fcf542a351b_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:22,078 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109507615ce716f403fb0302fcf542a351b_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:22,084 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=50 2024-11-09T17:26:22,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:22,085 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:22,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:22,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741971_1147 (size=12207) 2024-11-09T17:26:22,097 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/8e83e3cd7a9e48c182360dba27a3a433 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8e83e3cd7a9e48c182360dba27a3a433 2024-11-09T17:26:22,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092d8d70eaef3746f597005076503aabfc_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173181519/Put/seqid=0 2024-11-09T17:26:22,110 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into 8e83e3cd7a9e48c182360dba27a3a433(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:22,110 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:22,110 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=13, startTime=1731173182044; duration=0sec 2024-11-09T17:26:22,110 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:22,110 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:22,110 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:22,112 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:22,112 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:22,112 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:22,112 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/ee8e802969fd4afb8bc9522901e048e3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=33.0 K 2024-11-09T17:26:22,113 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ee8e802969fd4afb8bc9522901e048e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731173178585 2024-11-09T17:26:22,113 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e6e66adc88db486fb0ebfa237d1a3224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173179224 2024-11-09T17:26:22,113 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2efbff6657b94c688b75e1da7824921d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173180362 2024-11-09T17:26:22,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741972_1148 (size=4469) 2024-11-09T17:26:22,136 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#126 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:22,137 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/8bb93896b9b94c2dbde492d8df1cf546 is 50, key is test_row_0/C:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:22,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:22,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741973_1149 (size=12154) 2024-11-09T17:26:22,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741974_1150 (size=12207) 2024-11-09T17:26:22,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173242153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173242154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173242154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173242154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173242155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,165 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/8bb93896b9b94c2dbde492d8df1cf546 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/8bb93896b9b94c2dbde492d8df1cf546 2024-11-09T17:26:22,172 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into 8bb93896b9b94c2dbde492d8df1cf546(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:22,173 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:22,173 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=13, startTime=1731173182045; duration=0sec 2024-11-09T17:26:22,173 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:22,173 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:22,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173242261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173242261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173242263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173242263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173242263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173242465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173242465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173242466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173242466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173242466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,532 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#124 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:22,533 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/dbd56c5c23024e9fa4ffd93b7cfeb427 is 175, key is test_row_0/A:col10/1731173179224/Put/seqid=0 2024-11-09T17:26:22,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:22,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741975_1151 (size=31268) 2024-11-09T17:26:22,587 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092d8d70eaef3746f597005076503aabfc_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092d8d70eaef3746f597005076503aabfc_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:22,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/ecdb89911d0a42d68b1680b20528a0cf, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:22,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/ecdb89911d0a42d68b1680b20528a0cf is 175, key is test_row_0/A:col10/1731173181519/Put/seqid=0 2024-11-09T17:26:22,594 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/dbd56c5c23024e9fa4ffd93b7cfeb427 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427 2024-11-09T17:26:22,600 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into dbd56c5c23024e9fa4ffd93b7cfeb427(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:22,600 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:22,600 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=13, startTime=1731173182044; duration=0sec 2024-11-09T17:26:22,600 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:22,600 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:22,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741976_1152 (size=30955) 2024-11-09T17:26:22,608 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/ecdb89911d0a42d68b1680b20528a0cf 2024-11-09T17:26:22,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e14458b185294d30afe75d45bffa0a98 is 50, key is test_row_0/B:col10/1731173181519/Put/seqid=0 2024-11-09T17:26:22,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741977_1153 (size=12001) 2024-11-09T17:26:22,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173242770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173242770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173242770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173242771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:22,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173242777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,033 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e14458b185294d30afe75d45bffa0a98 2024-11-09T17:26:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5c78bf98a73b4ebcb71790b1844ff10f is 50, key is test_row_0/C:col10/1731173181519/Put/seqid=0 2024-11-09T17:26:23,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741978_1154 (size=12001) 2024-11-09T17:26:23,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173243274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173243275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173243276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173243278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173243283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5c78bf98a73b4ebcb71790b1844ff10f 2024-11-09T17:26:23,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/ecdb89911d0a42d68b1680b20528a0cf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf 2024-11-09T17:26:23,461 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf, entries=150, sequenceid=116, filesize=30.2 K 2024-11-09T17:26:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/e14458b185294d30afe75d45bffa0a98 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98 2024-11-09T17:26:23,466 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:26:23,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5c78bf98a73b4ebcb71790b1844ff10f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f 2024-11-09T17:26:23,473 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:26:23,474 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 9b0fa22acef654074d43b8d7d6d68c6a in 1389ms, sequenceid=116, compaction requested=false 2024-11-09T17:26:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-09T17:26:23,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-11-09T17:26:23,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-09T17:26:23,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8510 sec 2024-11-09T17:26:23,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=49, table=TestAcidGuarantees in 1.8570 sec 2024-11-09T17:26:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-09T17:26:23,728 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 49 completed 2024-11-09T17:26:23,729 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees 2024-11-09T17:26:23,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:23,730 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:23,731 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=51, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:23,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:23,882 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:23,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=52 2024-11-09T17:26:23,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:23,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091e98c5aad43347a8a47024d19abddbe3_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173182149/Put/seqid=0 2024-11-09T17:26:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741979_1155 (size=12254) 2024-11-09T17:26:24,025 INFO [master/c28fd47d982f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T17:26:24,025 INFO [master/c28fd47d982f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T17:26:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:24,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:24,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:24,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173244293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173244294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173244295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173244296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173244297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:24,311 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091e98c5aad43347a8a47024d19abddbe3_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e98c5aad43347a8a47024d19abddbe3_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:24,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6537ab7f61e64942a6d6c14543ef89d1, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:24,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6537ab7f61e64942a6d6c14543ef89d1 is 175, key is test_row_0/A:col10/1731173182149/Put/seqid=0 2024-11-09T17:26:24,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741980_1156 (size=31055) 2024-11-09T17:26:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173244399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173244399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173244399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173244399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173244400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173244603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173244604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173244605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173244605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173244605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,718 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6537ab7f61e64942a6d6c14543ef89d1 2024-11-09T17:26:24,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/ba240b1966b048b0a4ae9b9711c13362 is 50, key is test_row_0/B:col10/1731173182149/Put/seqid=0 2024-11-09T17:26:24,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741981_1157 (size=12101) 2024-11-09T17:26:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:24,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173244908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173244908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173244909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173244909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:24,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173244911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/ba240b1966b048b0a4ae9b9711c13362 2024-11-09T17:26:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e4e02ae23e2a4f8b8e3facc57f994f8a is 50, key is test_row_0/C:col10/1731173182149/Put/seqid=0 2024-11-09T17:26:25,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741982_1158 (size=12101) 2024-11-09T17:26:25,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173245412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173245413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173245414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173245415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173245415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,552 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e4e02ae23e2a4f8b8e3facc57f994f8a 2024-11-09T17:26:25,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6537ab7f61e64942a6d6c14543ef89d1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1 2024-11-09T17:26:25,561 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1, entries=150, sequenceid=133, filesize=30.3 K 2024-11-09T17:26:25,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/ba240b1966b048b0a4ae9b9711c13362 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362 2024-11-09T17:26:25,569 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362, entries=150, sequenceid=133, filesize=11.8 K 2024-11-09T17:26:25,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/e4e02ae23e2a4f8b8e3facc57f994f8a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a 2024-11-09T17:26:25,580 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-09T17:26:25,581 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 9b0fa22acef654074d43b8d7d6d68c6a in 1698ms, sequenceid=133, compaction requested=true 2024-11-09T17:26:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:25,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=52}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=52 2024-11-09T17:26:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=52 2024-11-09T17:26:25,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=52, resume processing ppid=51 2024-11-09T17:26:25,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=52, ppid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8510 sec 2024-11-09T17:26:25,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=51, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=51, table=TestAcidGuarantees in 1.8550 sec 2024-11-09T17:26:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-09T17:26:25,834 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 51 completed 2024-11-09T17:26:25,835 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees 2024-11-09T17:26:25,836 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-09T17:26:25,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=53, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:25,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-09T17:26:25,991 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:25,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=54 2024-11-09T17:26:25,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:25,991 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:25,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:26,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f14ed9b3bdf940018bba1295f58e5f01_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173184296/Put/seqid=0 2024-11-09T17:26:26,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741983_1159 (size=12304) 2024-11-09T17:26:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:26,013 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f14ed9b3bdf940018bba1295f58e5f01_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f14ed9b3bdf940018bba1295f58e5f01_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:26,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/9f264372e5f34cf4b6faa7afe98e8384, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/9f264372e5f34cf4b6faa7afe98e8384 is 175, key is test_row_0/A:col10/1731173184296/Put/seqid=0 2024-11-09T17:26:26,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741984_1160 (size=31105) 2024-11-09T17:26:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-09T17:26:26,420 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/9f264372e5f34cf4b6faa7afe98e8384 2024-11-09T17:26:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:26,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:26,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f13998adee7c47878a5407354ebaaf47 is 50, key is test_row_0/B:col10/1731173184296/Put/seqid=0 2024-11-09T17:26:26,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173246430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173246431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173246432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173246432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173246433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741985_1161 (size=12151) 2024-11-09T17:26:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-09T17:26:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173246533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173246533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173246534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173246534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173246534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173246736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173246736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173246737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173246737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173246737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:26,836 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f13998adee7c47878a5407354ebaaf47 2024-11-09T17:26:26,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44ca469d175e4303addaea59983a1c21 is 50, key is test_row_0/C:col10/1731173184296/Put/seqid=0 2024-11-09T17:26:26,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741986_1162 (size=12151) 2024-11-09T17:26:26,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44ca469d175e4303addaea59983a1c21 2024-11-09T17:26:26,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/9f264372e5f34cf4b6faa7afe98e8384 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384 2024-11-09T17:26:26,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384, entries=150, sequenceid=155, filesize=30.4 K 2024-11-09T17:26:26,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/f13998adee7c47878a5407354ebaaf47 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47 2024-11-09T17:26:26,875 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47, entries=150, sequenceid=155, filesize=11.9 K 2024-11-09T17:26:26,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/44ca469d175e4303addaea59983a1c21 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21 2024-11-09T17:26:26,883 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21, entries=150, sequenceid=155, filesize=11.9 K 2024-11-09T17:26:26,884 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 9b0fa22acef654074d43b8d7d6d68c6a in 893ms, sequenceid=155, compaction requested=true 2024-11-09T17:26:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:26,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=54}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=54 2024-11-09T17:26:26,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=54 2024-11-09T17:26:26,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=54, resume processing ppid=53 2024-11-09T17:26:26,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=54, ppid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0510 sec 2024-11-09T17:26:26,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=53, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=53, table=TestAcidGuarantees in 1.0560 sec 2024-11-09T17:26:26,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-09T17:26:26,940 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 53 completed 2024-11-09T17:26:26,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees 2024-11-09T17:26:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:26,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:26,943 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=55, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:26,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:27,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T17:26:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:27,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:27,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:27,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:27,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091dcef657b30543d4b0bbc00fcc434073_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741987_1163 (size=17284) 2024-11-09T17:26:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173247060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173247060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173247062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173247063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173247063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,094 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173247164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173247164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173247164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173247167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173247167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:27,247 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173247368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173247368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173247368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173247370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173247374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,400 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:27,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091dcef657b30543d4b0bbc00fcc434073_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091dcef657b30543d4b0bbc00fcc434073_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:27,463 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/74dbac443e264acdad114db280bdf913, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:27,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/74dbac443e264acdad114db280bdf913 is 175, key is test_row_0/A:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:27,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741988_1164 (size=48389) 2024-11-09T17:26:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:27,552 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173247670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173247671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173247672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173247674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173247676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,705 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,857 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:27,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:27,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:27,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:27,870 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/74dbac443e264acdad114db280bdf913 2024-11-09T17:26:27,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/21af2fcd79ba4eca9462fafce5a74650 is 50, key is test_row_0/B:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:27,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741989_1165 (size=12151) 2024-11-09T17:26:28,010 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:28,163 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:28,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173248175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173248176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173248177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:28,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173248180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:28,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173248180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/21af2fcd79ba4eca9462fafce5a74650 2024-11-09T17:26:28,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/89645a1dc912491b8408f05f9f4d2d9f is 50, key is test_row_0/C:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:28,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741990_1166 (size=12151) 2024-11-09T17:26:28,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/89645a1dc912491b8408f05f9f4d2d9f 2024-11-09T17:26:28,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/74dbac443e264acdad114db280bdf913 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913 2024-11-09T17:26:28,310 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913, entries=250, sequenceid=172, filesize=47.3 K 2024-11-09T17:26:28,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/21af2fcd79ba4eca9462fafce5a74650 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650 2024-11-09T17:26:28,316 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:28,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:28,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] handler.RSProcedureHandler(58): pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=56 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T17:26:28,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=56 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:28,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/89645a1dc912491b8408f05f9f4d2d9f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f 2024-11-09T17:26:28,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T17:26:28,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 9b0fa22acef654074d43b8d7d6d68c6a in 1287ms, sequenceid=172, compaction requested=true 2024-11-09T17:26:28,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:28,329 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:28,329 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:28,331 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172772 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T17:26:28,331 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T17:26:28,331 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:28,331 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:28,331 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,331 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=168.7 K 2024-11-09T17:26:28,331 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,331 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913] 2024-11-09T17:26:28,331 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,332 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8e83e3cd7a9e48c182360dba27a3a433, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=59.2 K 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting dbd56c5c23024e9fa4ffd93b7cfeb427, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173179224 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8e83e3cd7a9e48c182360dba27a3a433, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173179224 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting ecdb89911d0a42d68b1680b20528a0cf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173181519 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e14458b185294d30afe75d45bffa0a98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173181519 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6537ab7f61e64942a6d6c14543ef89d1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731173182149 2024-11-09T17:26:28,332 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ba240b1966b048b0a4ae9b9711c13362, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731173182149 2024-11-09T17:26:28,333 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9f264372e5f34cf4b6faa7afe98e8384, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731173184294 2024-11-09T17:26:28,333 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f13998adee7c47878a5407354ebaaf47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731173184294 2024-11-09T17:26:28,333 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 74dbac443e264acdad114db280bdf913, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:28,334 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 21af2fcd79ba4eca9462fafce5a74650, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:28,346 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:28,347 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/a395f232c7334547950eb6f9768b0724 is 50, key is test_row_0/B:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:28,355 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:28,366 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110980a39cd015794ec3a8cd038f14030e4b_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:28,369 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110980a39cd015794ec3a8cd038f14030e4b_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:28,369 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110980a39cd015794ec3a8cd038f14030e4b_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:28,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741991_1167 (size=12527) 2024-11-09T17:26:28,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741992_1168 (size=4469) 2024-11-09T17:26:28,470 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:28,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=56 2024-11-09T17:26:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,470 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:28,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ea5b8c834f56405aa4c7efb50132eac5_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173187061/Put/seqid=0 2024-11-09T17:26:28,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741993_1169 (size=12304) 2024-11-09T17:26:28,786 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/a395f232c7334547950eb6f9768b0724 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/a395f232c7334547950eb6f9768b0724 2024-11-09T17:26:28,790 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#139 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:28,790 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/efdb0e1236074a80bc8bc963fc1ed5fd is 175, key is test_row_0/A:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:28,792 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into a395f232c7334547950eb6f9768b0724(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:28,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:28,793 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=11, startTime=1731173188329; duration=0sec 2024-11-09T17:26:28,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:28,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:28,793 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T17:26:28,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741994_1170 (size=31481) 2024-11-09T17:26:28,795 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60611 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T17:26:28,795 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:28,795 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:28,795 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/8bb93896b9b94c2dbde492d8df1cf546, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=59.2 K 2024-11-09T17:26:28,796 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8bb93896b9b94c2dbde492d8df1cf546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731173179224 2024-11-09T17:26:28,796 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 5c78bf98a73b4ebcb71790b1844ff10f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173181519 2024-11-09T17:26:28,796 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e4e02ae23e2a4f8b8e3facc57f994f8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731173182149 2024-11-09T17:26:28,802 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 44ca469d175e4303addaea59983a1c21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731173184294 2024-11-09T17:26:28,803 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 89645a1dc912491b8408f05f9f4d2d9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:28,805 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/efdb0e1236074a80bc8bc963fc1ed5fd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd 2024-11-09T17:26:28,810 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into efdb0e1236074a80bc8bc963fc1ed5fd(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:28,810 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:28,810 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=11, startTime=1731173188329; duration=0sec 2024-11-09T17:26:28,810 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:28,810 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:28,814 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:28,815 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/adedcecd764f479cb639e9a38bfe2f94 is 50, key is test_row_0/C:col10/1731173186431/Put/seqid=0 2024-11-09T17:26:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741995_1171 (size=12527) 2024-11-09T17:26:28,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:28,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ea5b8c834f56405aa4c7efb50132eac5_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ea5b8c834f56405aa4c7efb50132eac5_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:28,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6fb715c1ecad45f5bd2e7b566f7458dc, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:28,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6fb715c1ecad45f5bd2e7b566f7458dc is 175, key is test_row_0/A:col10/1731173187061/Put/seqid=0 2024-11-09T17:26:28,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741996_1172 (size=31105) 2024-11-09T17:26:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:29,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173249193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173249192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173249196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173249196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173249202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,225 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/adedcecd764f479cb639e9a38bfe2f94 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/adedcecd764f479cb639e9a38bfe2f94 2024-11-09T17:26:29,229 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 5 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into adedcecd764f479cb639e9a38bfe2f94(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:29,229 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:29,230 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=11, startTime=1731173188329; duration=0sec 2024-11-09T17:26:29,230 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:29,230 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173249297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173249297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173249299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173249299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,303 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6fb715c1ecad45f5bd2e7b566f7458dc 2024-11-09T17:26:29,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173249304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/05d79568b5cf43228fcea1db55cfef30 is 50, key is test_row_0/B:col10/1731173187061/Put/seqid=0 2024-11-09T17:26:29,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741997_1173 (size=12151) 2024-11-09T17:26:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173249498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173249498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173249500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173249501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173249506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,715 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/05d79568b5cf43228fcea1db55cfef30 2024-11-09T17:26:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f1f8d58f66224a3984178ac3ba83a789 is 50, key is test_row_0/C:col10/1731173187061/Put/seqid=0 2024-11-09T17:26:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741998_1174 (size=12151) 2024-11-09T17:26:29,748 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f1f8d58f66224a3984178ac3ba83a789 2024-11-09T17:26:29,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/6fb715c1ecad45f5bd2e7b566f7458dc as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc 2024-11-09T17:26:29,759 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc, entries=150, sequenceid=192, filesize=30.4 K 2024-11-09T17:26:29,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/05d79568b5cf43228fcea1db55cfef30 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30 2024-11-09T17:26:29,767 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T17:26:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/f1f8d58f66224a3984178ac3ba83a789 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789 2024-11-09T17:26:29,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T17:26:29,781 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 9b0fa22acef654074d43b8d7d6d68c6a in 1311ms, sequenceid=192, compaction requested=false 2024-11-09T17:26:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:29,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=56}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=56 2024-11-09T17:26:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=56 2024-11-09T17:26:29,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=56, resume processing ppid=55 2024-11-09T17:26:29,786 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=56, ppid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8390 sec 2024-11-09T17:26:29,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=55, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=55, table=TestAcidGuarantees in 2.8450 sec 2024-11-09T17:26:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:29,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T17:26:29,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:29,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:29,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:29,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:29,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:29,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:29,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109150bde2b09d34f208b65e446932d72fc_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:29,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173249823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173249824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173249824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173249825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173249825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741999_1175 (size=17284) 2024-11-09T17:26:29,834 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:29,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109150bde2b09d34f208b65e446932d72fc_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109150bde2b09d34f208b65e446932d72fc_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:29,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/5c4ed30afd7a45f59534b8033ad37cb8, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:29,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/5c4ed30afd7a45f59534b8033ad37cb8 is 175, key is test_row_0/A:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:29,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742000_1176 (size=48389) 2024-11-09T17:26:29,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173249928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173249930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173249931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173249931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:29,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:29,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173249931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173250132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173250132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173250133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173250133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173250134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,257 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/5c4ed30afd7a45f59534b8033ad37cb8 2024-11-09T17:26:30,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/d885da92228f4dbfbffe71cd02628eee is 50, key is test_row_0/B:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:30,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742001_1177 (size=12151) 2024-11-09T17:26:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173250436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173250436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173250436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173250436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173250436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/d885da92228f4dbfbffe71cd02628eee 2024-11-09T17:26:30,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/297307066c994239bb9605af385b44ee is 50, key is test_row_0/C:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:30,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742002_1178 (size=12151) 2024-11-09T17:26:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173250940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173250942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173250943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173250943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:30,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173250943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-09T17:26:31,046 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 55 completed 2024-11-09T17:26:31,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees 2024-11-09T17:26:31,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:31,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:31,049 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=57, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:31,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:31,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/297307066c994239bb9605af385b44ee 2024-11-09T17:26:31,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/5c4ed30afd7a45f59534b8033ad37cb8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8 2024-11-09T17:26:31,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8, entries=250, sequenceid=214, filesize=47.3 K 2024-11-09T17:26:31,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/d885da92228f4dbfbffe71cd02628eee as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee 2024-11-09T17:26:31,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T17:26:31,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/297307066c994239bb9605af385b44ee as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee 2024-11-09T17:26:31,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee, entries=150, sequenceid=214, filesize=11.9 K 2024-11-09T17:26:31,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 9b0fa22acef654074d43b8d7d6d68c6a in 1295ms, sequenceid=214, compaction requested=true 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:31,099 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:31,099 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:31,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:31,100 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:31,100 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:31,100 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:31,100 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:31,100 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:31,100 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:31,100 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/a395f232c7334547950eb6f9768b0724, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.0 K 2024-11-09T17:26:31,100 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=108.4 K 2024-11-09T17:26:31,100 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:31,100 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8] 2024-11-09T17:26:31,101 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a395f232c7334547950eb6f9768b0724, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:31,101 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting efdb0e1236074a80bc8bc963fc1ed5fd, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:31,101 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 05d79568b5cf43228fcea1db55cfef30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731173187051 2024-11-09T17:26:31,101 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6fb715c1ecad45f5bd2e7b566f7458dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731173187051 2024-11-09T17:26:31,101 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d885da92228f4dbfbffe71cd02628eee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:31,102 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5c4ed30afd7a45f59534b8033ad37cb8, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:31,108 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#147 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:31,108 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/61b933892b644f4987b6c4d800a0e611 is 50, key is test_row_0/B:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:31,109 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:31,112 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110979ee49c82c134449962d61b902192984_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:31,114 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110979ee49c82c134449962d61b902192984_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:31,114 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110979ee49c82c134449962d61b902192984_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:31,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742004_1180 (size=4469) 2024-11-09T17:26:31,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742003_1179 (size=12629) 2024-11-09T17:26:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:31,200 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=58 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:31,201 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:31,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:31,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095c3e11dbc51f4510af5018e783468bcb_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173189815/Put/seqid=0 2024-11-09T17:26:31,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742005_1181 (size=12304) 2024-11-09T17:26:31,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:31,520 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#148 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:31,521 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/38c59810c2f94461bdc5fe5ee82dcb61 is 175, key is test_row_0/A:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:31,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742006_1182 (size=31583) 2024-11-09T17:26:31,525 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/61b933892b644f4987b6c4d800a0e611 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/61b933892b644f4987b6c4d800a0e611 2024-11-09T17:26:31,530 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into 61b933892b644f4987b6c4d800a0e611(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:31,530 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:31,530 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=13, startTime=1731173191099; duration=0sec 2024-11-09T17:26:31,530 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:31,530 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:31,530 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:31,532 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:31,532 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:31,532 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:31,532 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/adedcecd764f479cb639e9a38bfe2f94, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.0 K 2024-11-09T17:26:31,532 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting adedcecd764f479cb639e9a38bfe2f94, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173186430 2024-11-09T17:26:31,533 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f1f8d58f66224a3984178ac3ba83a789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731173187051 2024-11-09T17:26:31,533 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 297307066c994239bb9605af385b44ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:31,539 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:31,540 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/1606febc243641a9a2344f1216b07b8e is 50, key is test_row_0/C:col10/1731173189194/Put/seqid=0 2024-11-09T17:26:31,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742007_1183 (size=12629) 2024-11-09T17:26:31,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:31,618 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095c3e11dbc51f4510af5018e783468bcb_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c3e11dbc51f4510af5018e783468bcb_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b62067e48dc245f4a7699d2b3da16726, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:31,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b62067e48dc245f4a7699d2b3da16726 is 175, key is test_row_0/A:col10/1731173189815/Put/seqid=0 2024-11-09T17:26:31,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742008_1184 (size=31105) 2024-11-09T17:26:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:31,930 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/38c59810c2f94461bdc5fe5ee82dcb61 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61 2024-11-09T17:26:31,935 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into 38c59810c2f94461bdc5fe5ee82dcb61(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:31,935 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:31,935 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=13, startTime=1731173191099; duration=0sec 2024-11-09T17:26:31,935 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:31,935 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:31,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:31,954 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/1606febc243641a9a2344f1216b07b8e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/1606febc243641a9a2344f1216b07b8e 2024-11-09T17:26:31,960 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into 1606febc243641a9a2344f1216b07b8e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:31,960 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:31,960 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=13, startTime=1731173191099; duration=0sec 2024-11-09T17:26:31,960 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:31,960 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:31,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173251962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173251963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:31,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173251963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173251964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:31,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:31,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173251965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b62067e48dc245f4a7699d2b3da16726 2024-11-09T17:26:32,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/8b865fd9a79c4dc18bf0f36574f83a96 is 50, key is test_row_0/B:col10/1731173189815/Put/seqid=0 2024-11-09T17:26:32,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742009_1185 (size=12151) 2024-11-09T17:26:32,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173252066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173252066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173252066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173252066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173252067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:32,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173252267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173252268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173252268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173252269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173252271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,436 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/8b865fd9a79c4dc18bf0f36574f83a96 2024-11-09T17:26:32,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/b8c5bf4e15004a5a9e0e65245b8f5415 is 50, key is test_row_0/C:col10/1731173189815/Put/seqid=0 2024-11-09T17:26:32,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742010_1186 (size=12151) 2024-11-09T17:26:32,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173252570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173252571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173252575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173252575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:32,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173252575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:32,852 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/b8c5bf4e15004a5a9e0e65245b8f5415 2024-11-09T17:26:32,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b62067e48dc245f4a7699d2b3da16726 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726 2024-11-09T17:26:32,861 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726, entries=150, sequenceid=232, filesize=30.4 K 2024-11-09T17:26:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/8b865fd9a79c4dc18bf0f36574f83a96 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96 2024-11-09T17:26:32,865 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T17:26:32,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/b8c5bf4e15004a5a9e0e65245b8f5415 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415 2024-11-09T17:26:32,870 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415, entries=150, sequenceid=232, filesize=11.9 K 2024-11-09T17:26:32,871 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 9b0fa22acef654074d43b8d7d6d68c6a in 1670ms, sequenceid=232, compaction requested=false 2024-11-09T17:26:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:32,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-11-09T17:26:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=58 2024-11-09T17:26:32,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T17:26:32,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=58, ppid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-09T17:26:32,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=57, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=57, table=TestAcidGuarantees in 1.8270 sec 2024-11-09T17:26:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:33,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:33,078 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:33,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110925b35bcb07d347d7b3789b91ca83319e_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:33,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742011_1187 (size=14794) 2024-11-09T17:26:33,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173253090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173253091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173253091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173253091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173253092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-09T17:26:33,152 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 57 completed 2024-11-09T17:26:33,153 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees 2024-11-09T17:26:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:33,154 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:33,155 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=59, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:33,155 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:33,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173253194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173253195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173253195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173253195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173253196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:33,306 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173253396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173253396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173253399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173253399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173253399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:33,458 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,491 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:33,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110925b35bcb07d347d7b3789b91ca83319e_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925b35bcb07d347d7b3789b91ca83319e_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:33,495 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b3024d1468454e18b2a131dd3705d0a3, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:33,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b3024d1468454e18b2a131dd3705d0a3 is 175, key is test_row_0/A:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742012_1188 (size=39749) 2024-11-09T17:26:33,611 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173253700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173253701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173253701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173253703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173253703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:33,764 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:33,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,901 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b3024d1468454e18b2a131dd3705d0a3 2024-11-09T17:26:33,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/bc45a2f236d1412aa911bc1df1777657 is 50, key is test_row_0/B:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:33,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742013_1189 (size=12151) 2024-11-09T17:26:33,916 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:33,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:33,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:33,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,068 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173254202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:34,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173254204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:34,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173254204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173254208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:34,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173254211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,221 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:34,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:34,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/bc45a2f236d1412aa911bc1df1777657 2024-11-09T17:26:34,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/821c8339edf2424fba9a7d5db2bc6ab6 is 50, key is test_row_0/C:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:34,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742014_1190 (size=12151) 2024-11-09T17:26:34,373 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,526 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,679 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] handler.RSProcedureHandler(58): pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=60 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=60 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:34,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/821c8339edf2424fba9a7d5db2bc6ab6 2024-11-09T17:26:34,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/b3024d1468454e18b2a131dd3705d0a3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3 2024-11-09T17:26:34,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3, entries=200, sequenceid=254, filesize=38.8 K 2024-11-09T17:26:34,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/bc45a2f236d1412aa911bc1df1777657 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657 2024-11-09T17:26:34,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657, entries=150, sequenceid=254, filesize=11.9 K 2024-11-09T17:26:34,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/821c8339edf2424fba9a7d5db2bc6ab6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6 2024-11-09T17:26:34,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6, entries=150, sequenceid=254, filesize=11.9 K 2024-11-09T17:26:34,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 9b0fa22acef654074d43b8d7d6d68c6a in 1662ms, sequenceid=254, compaction requested=true 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:34,739 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:34,739 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:34,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:34,740 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:34,740 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:34,740 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:34,740 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:34,740 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,740 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,741 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=100.0 K 2024-11-09T17:26:34,741 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/61b933892b644f4987b6c4d800a0e611, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.1 K 2024-11-09T17:26:34,741 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,741 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3] 2024-11-09T17:26:34,741 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 61b933892b644f4987b6c4d800a0e611, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:34,741 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 38c59810c2f94461bdc5fe5ee82dcb61, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:34,741 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8b865fd9a79c4dc18bf0f36574f83a96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731173189815 2024-11-09T17:26:34,741 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b62067e48dc245f4a7699d2b3da16726, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731173189815 2024-11-09T17:26:34,742 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b3024d1468454e18b2a131dd3705d0a3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191961 2024-11-09T17:26:34,742 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting bc45a2f236d1412aa911bc1df1777657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191962 2024-11-09T17:26:34,755 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:34,756 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:34,756 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/798b4bbaf0ad42a39c4b686805488c3f is 50, key is test_row_0/B:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:34,763 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411091a67a1e77e6c479788ea93f8eb7122e3_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:34,765 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411091a67a1e77e6c479788ea93f8eb7122e3_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:34,765 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091a67a1e77e6c479788ea93f8eb7122e3_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:34,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742015_1191 (size=12731) 2024-11-09T17:26:34,773 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/798b4bbaf0ad42a39c4b686805488c3f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/798b4bbaf0ad42a39c4b686805488c3f 2024-11-09T17:26:34,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742016_1192 (size=4469) 2024-11-09T17:26:34,778 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#157 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:34,778 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/4ac62744b83c40a9812a694a54dac931 is 175, key is test_row_0/A:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:34,778 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into 798b4bbaf0ad42a39c4b686805488c3f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:34,778 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:34,778 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=13, startTime=1731173194739; duration=0sec 2024-11-09T17:26:34,779 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:34,779 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:34,779 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:34,780 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:34,780 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:34,780 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,780 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/1606febc243641a9a2344f1216b07b8e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.1 K 2024-11-09T17:26:34,781 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1606febc243641a9a2344f1216b07b8e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731173189192 2024-11-09T17:26:34,781 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting b8c5bf4e15004a5a9e0e65245b8f5415, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731173189815 2024-11-09T17:26:34,781 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 821c8339edf2424fba9a7d5db2bc6ab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191962 2024-11-09T17:26:34,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742017_1193 (size=31685) 2024-11-09T17:26:34,790 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/4ac62744b83c40a9812a694a54dac931 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931 2024-11-09T17:26:34,795 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:34,796 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/a51b830b109f4901be5d8ce2ed7e8bb7 is 50, key is test_row_0/C:col10/1731173191964/Put/seqid=0 2024-11-09T17:26:34,798 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into 4ac62744b83c40a9812a694a54dac931(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:34,798 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:34,798 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=13, startTime=1731173194739; duration=0sec 2024-11-09T17:26:34,798 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:34,798 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:34,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742018_1194 (size=12731) 2024-11-09T17:26:34,813 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/a51b830b109f4901be5d8ce2ed7e8bb7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/a51b830b109f4901be5d8ce2ed7e8bb7 2024-11-09T17:26:34,820 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into a51b830b109f4901be5d8ce2ed7e8bb7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:34,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:34,820 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=13, startTime=1731173194739; duration=0sec 2024-11-09T17:26:34,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:34,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:34,831 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:34,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=60 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:34,832 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:34,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099980ae6a58204ca5857da60ea1312bc4_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173193090/Put/seqid=0 2024-11-09T17:26:34,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742019_1195 (size=12454) 2024-11-09T17:26:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:35,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173255221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173255221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173255221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173255221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173255222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:35,247 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099980ae6a58204ca5857da60ea1312bc4_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099980ae6a58204ca5857da60ea1312bc4_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:35,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e01337d40d8144458411d22002c6ba36, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:35,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e01337d40d8144458411d22002c6ba36 is 175, key is test_row_0/A:col10/1731173193090/Put/seqid=0 2024-11-09T17:26:35,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742020_1196 (size=31255) 2024-11-09T17:26:35,253 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e01337d40d8144458411d22002c6ba36 2024-11-09T17:26:35,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:35,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/95eeb3b1e32247348dd217c9d7706d40 is 50, key is test_row_0/B:col10/1731173193090/Put/seqid=0 2024-11-09T17:26:35,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742021_1197 (size=12301) 2024-11-09T17:26:35,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173255324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173255324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173255324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173255325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173255325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173255526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173255526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173255526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173255527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173255527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,668 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/95eeb3b1e32247348dd217c9d7706d40 2024-11-09T17:26:35,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/420f6eed05054db3aa4df0aa48078ab5 is 50, key is test_row_0/C:col10/1731173193090/Put/seqid=0 2024-11-09T17:26:35,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742022_1198 (size=12301) 2024-11-09T17:26:35,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173255828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173255828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173255828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173255829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:35,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:35,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173255829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,093 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/420f6eed05054db3aa4df0aa48078ab5 2024-11-09T17:26:36,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e01337d40d8144458411d22002c6ba36 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36 2024-11-09T17:26:36,101 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36, entries=150, sequenceid=274, filesize=30.5 K 2024-11-09T17:26:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/95eeb3b1e32247348dd217c9d7706d40 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40 2024-11-09T17:26:36,106 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T17:26:36,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/420f6eed05054db3aa4df0aa48078ab5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5 2024-11-09T17:26:36,110 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T17:26:36,111 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 9b0fa22acef654074d43b8d7d6d68c6a in 1279ms, sequenceid=274, compaction requested=false 2024-11-09T17:26:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:36,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-09T17:26:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-11-09T17:26:36,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-11-09T17:26:36,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9570 sec 2024-11-09T17:26:36,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=59, table=TestAcidGuarantees in 2.9600 sec 2024-11-09T17:26:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:36,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:36,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b53777e3f6c745bd908d5959dcabb602_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:36,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742023_1199 (size=14994) 2024-11-09T17:26:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173256354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173256353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173256354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173256356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173256357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173256458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173256458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173256458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173256458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173256462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,581 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:26:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173256660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173256660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173256660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173256665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173256671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:36,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b53777e3f6c745bd908d5959dcabb602_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b53777e3f6c745bd908d5959dcabb602_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:36,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e2303363b89c4cc29434f1fd233f799e, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:36,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e2303363b89c4cc29434f1fd233f799e is 175, key is test_row_0/A:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:36,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742024_1200 (size=39949) 2024-11-09T17:26:36,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173256975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173256975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173256975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173256975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173256975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,166 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e2303363b89c4cc29434f1fd233f799e 2024-11-09T17:26:37,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/726164f748a24358b69ace305f7cc7da is 50, key is test_row_0/B:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:37,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742025_1201 (size=12301) 2024-11-09T17:26:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-09T17:26:37,258 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 59 completed 2024-11-09T17:26:37,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees 2024-11-09T17:26:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:37,261 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:37,262 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=61, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:37,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:37,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:37,400 DEBUG [Thread-627 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x517ff977 to 127.0.0.1:57222 2024-11-09T17:26:37,400 DEBUG [Thread-627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:37,403 DEBUG [Thread-625 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x247c0c93 to 127.0.0.1:57222 2024-11-09T17:26:37,403 DEBUG [Thread-625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:37,404 DEBUG [Thread-629 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3448d233 to 127.0.0.1:57222 2024-11-09T17:26:37,404 DEBUG [Thread-629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:37,406 DEBUG [Thread-631 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a11164b to 127.0.0.1:57222 2024-11-09T17:26:37,406 DEBUG [Thread-631 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:37,416 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,416 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-09T17:26:37,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:37,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33962 deadline: 1731173257478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:37,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33904 deadline: 1731173257480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:37,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33938 deadline: 1731173257480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:37,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33906 deadline: 1731173257481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:37,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33934 deadline: 1731173257483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:37,568 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-09T17:26:37,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:37,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/726164f748a24358b69ace305f7cc7da 2024-11-09T17:26:37,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/0cc907bc0c654863a996e532de6b5a59 is 50, key is test_row_0/C:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:37,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742026_1202 (size=12301) 2024-11-09T17:26:37,722 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-09T17:26:37,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:37,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:37,878 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:37,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-09T17:26:37,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:37,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] handler.RSProcedureHandler(58): pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=62 java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=62 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:38,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/0cc907bc0c654863a996e532de6b5a59 2024-11-09T17:26:38,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/e2303363b89c4cc29434f1fd233f799e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e 2024-11-09T17:26:38,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e, entries=200, sequenceid=297, filesize=39.0 K 2024-11-09T17:26:38,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/726164f748a24358b69ace305f7cc7da as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da 2024-11-09T17:26:38,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T17:26:38,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/0cc907bc0c654863a996e532de6b5a59 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59 2024-11-09T17:26:38,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59, entries=150, sequenceid=297, filesize=12.0 K 2024-11-09T17:26:38,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 9b0fa22acef654074d43b8d7d6d68c6a in 1686ms, sequenceid=297, compaction requested=true 2024-11-09T17:26:38,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:38,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:38,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:38,020 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:38,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:38,020 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:38,020 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:38,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 9b0fa22acef654074d43b8d7d6d68c6a:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:38,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/B is initiating minor compaction (all files) 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/A is initiating minor compaction (all files) 2024-11-09T17:26:38,022 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/B in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:38,022 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/A in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:38,022 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/798b4bbaf0ad42a39c4b686805488c3f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.5 K 2024-11-09T17:26:38,022 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=100.5 K 2024-11-09T17:26:38,022 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e] 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 798b4bbaf0ad42a39c4b686805488c3f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191962 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4ac62744b83c40a9812a694a54dac931, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191962 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 95eeb3b1e32247348dd217c9d7706d40, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173193090 2024-11-09T17:26:38,022 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e01337d40d8144458411d22002c6ba36, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173193090 2024-11-09T17:26:38,023 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 726164f748a24358b69ace305f7cc7da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731173196334 2024-11-09T17:26:38,023 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e2303363b89c4cc29434f1fd233f799e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731173195220 2024-11-09T17:26:38,030 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:38,031 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/7ca85e11d86246b2a97ef349b5e6d38c is 50, key is test_row_0/B:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:38,033 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:38,034 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:38,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=62 2024-11-09T17:26:38,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:38,035 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T17:26:38,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:38,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:38,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:38,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:38,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:38,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:38,036 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411095cbd674008974528befdd7f645bef2cd_9b0fa22acef654074d43b8d7d6d68c6a store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:38,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742027_1203 (size=12983) 2024-11-09T17:26:38,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b612a760d8ec438b974d686592fc8f6b_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173196352/Put/seqid=0 2024-11-09T17:26:38,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742028_1204 (size=9914) 2024-11-09T17:26:38,063 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411095cbd674008974528befdd7f645bef2cd_9b0fa22acef654074d43b8d7d6d68c6a, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:38,064 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095cbd674008974528befdd7f645bef2cd_9b0fa22acef654074d43b8d7d6d68c6a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:38,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742029_1205 (size=4469) 2024-11-09T17:26:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:38,446 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/7ca85e11d86246b2a97ef349b5e6d38c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7ca85e11d86246b2a97ef349b5e6d38c 2024-11-09T17:26:38,453 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/B of 9b0fa22acef654074d43b8d7d6d68c6a into 7ca85e11d86246b2a97ef349b5e6d38c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:38,453 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:38,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:38,453 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/B, priority=13, startTime=1731173198020; duration=0sec 2024-11-09T17:26:38,453 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:38,453 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:B 2024-11-09T17:26:38,453 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:38,454 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:38,454 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): 9b0fa22acef654074d43b8d7d6d68c6a/C is initiating minor compaction (all files) 2024-11-09T17:26:38,454 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of 9b0fa22acef654074d43b8d7d6d68c6a/C in TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:38,454 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/a51b830b109f4901be5d8ce2ed7e8bb7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp, totalSize=36.5 K 2024-11-09T17:26:38,454 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a51b830b109f4901be5d8ce2ed7e8bb7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731173191962 2024-11-09T17:26:38,455 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 420f6eed05054db3aa4df0aa48078ab5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173193090 2024-11-09T17:26:38,455 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0cc907bc0c654863a996e532de6b5a59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731173196334 2024-11-09T17:26:38,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b612a760d8ec438b974d686592fc8f6b_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b612a760d8ec438b974d686592fc8f6b_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/7580ab72ebae4029a4633335f9b75154, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:38,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/7580ab72ebae4029a4633335f9b75154 is 175, key is test_row_0/A:col10/1731173196352/Put/seqid=0 2024-11-09T17:26:38,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742030_1206 (size=22561) 2024-11-09T17:26:38,465 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:38,465 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/c21041ddddb24f51892f1288a1d26698 is 50, key is test_row_0/C:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:38,468 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 9b0fa22acef654074d43b8d7d6d68c6a#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:38,469 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/95cb5d8c277a4983aef81451d744d511 is 175, key is test_row_0/A:col10/1731173196334/Put/seqid=0 2024-11-09T17:26:38,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742031_1207 (size=12983) 2024-11-09T17:26:38,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742032_1208 (size=31937) 2024-11-09T17:26:38,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:38,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. as already flushing 2024-11-09T17:26:38,482 DEBUG [Thread-620 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x42e904d8 to 127.0.0.1:57222 2024-11-09T17:26:38,482 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:38,484 DEBUG [Thread-614 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7cae6c5c to 127.0.0.1:57222 2024-11-09T17:26:38,484 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:38,485 DEBUG [Thread-618 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b44b1e5 to 127.0.0.1:57222 2024-11-09T17:26:38,485 DEBUG [Thread-618 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:38,487 DEBUG [Thread-622 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a4c53ed to 127.0.0.1:57222 2024-11-09T17:26:38,487 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:38,493 DEBUG [Thread-616 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c820ef9 to 127.0.0.1:57222 2024-11-09T17:26:38,493 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:38,863 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/7580ab72ebae4029a4633335f9b75154 2024-11-09T17:26:38,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/7fb78d4a73d442c48365792849d87cd2 is 50, key is test_row_0/B:col10/1731173196352/Put/seqid=0 2024-11-09T17:26:38,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742033_1209 (size=9857) 2024-11-09T17:26:38,875 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/c21041ddddb24f51892f1288a1d26698 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/c21041ddddb24f51892f1288a1d26698 2024-11-09T17:26:38,877 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/95cb5d8c277a4983aef81451d744d511 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/95cb5d8c277a4983aef81451d744d511 2024-11-09T17:26:38,879 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/C of 9b0fa22acef654074d43b8d7d6d68c6a into c21041ddddb24f51892f1288a1d26698(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:38,880 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:38,880 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/C, priority=13, startTime=1731173198020; duration=0sec 2024-11-09T17:26:38,880 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:38,880 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:C 2024-11-09T17:26:38,881 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in 9b0fa22acef654074d43b8d7d6d68c6a/A of 9b0fa22acef654074d43b8d7d6d68c6a into 95cb5d8c277a4983aef81451d744d511(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:38,881 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:38,882 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a., storeName=9b0fa22acef654074d43b8d7d6d68c6a/A, priority=13, startTime=1731173198020; duration=0sec 2024-11-09T17:26:38,882 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:38,882 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 9b0fa22acef654074d43b8d7d6d68c6a:A 2024-11-09T17:26:39,276 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/7fb78d4a73d442c48365792849d87cd2 2024-11-09T17:26:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5ad0fc8009934bbe842052975d439637 is 50, key is test_row_0/C:col10/1731173196352/Put/seqid=0 2024-11-09T17:26:39,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742034_1210 (size=9857) 2024-11-09T17:26:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:39,696 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5ad0fc8009934bbe842052975d439637 2024-11-09T17:26:39,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/7580ab72ebae4029a4633335f9b75154 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/7580ab72ebae4029a4633335f9b75154 2024-11-09T17:26:39,712 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/7580ab72ebae4029a4633335f9b75154, entries=100, sequenceid=310, filesize=22.0 K 2024-11-09T17:26:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/7fb78d4a73d442c48365792849d87cd2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7fb78d4a73d442c48365792849d87cd2 2024-11-09T17:26:39,720 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7fb78d4a73d442c48365792849d87cd2, entries=100, sequenceid=310, filesize=9.6 K 2024-11-09T17:26:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/5ad0fc8009934bbe842052975d439637 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5ad0fc8009934bbe842052975d439637 2024-11-09T17:26:39,726 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5ad0fc8009934bbe842052975d439637, entries=100, sequenceid=310, filesize=9.6 K 2024-11-09T17:26:39,727 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=33.54 KB/34350 for 9b0fa22acef654074d43b8d7d6d68c6a in 1692ms, sequenceid=310, compaction requested=false 2024-11-09T17:26:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.HRegion(2538): Flush status journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-09T17:26:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-11-09T17:26:39,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-09T17:26:39,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4670 sec 2024-11-09T17:26:39,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=61, table=TestAcidGuarantees in 2.4710 sec 2024-11-09T17:26:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-09T17:26:41,371 INFO [Thread-624 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 61 completed 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-09T17:26:41,371 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6015 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 5888 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2493 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7479 rows 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2500 2024-11-09T17:26:41,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7500 rows 2024-11-09T17:26:41,372 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:26:41,372 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a9b9802 to 127.0.0.1:57222 2024-11-09T17:26:41,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:26:41,377 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:26:41,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:26:41,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:41,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:41,382 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173201381"}]},"ts":"1731173201381"} 2024-11-09T17:26:41,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:26:41,430 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:26:41,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:26:41,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, UNASSIGN}] 2024-11-09T17:26:41,436 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=65, ppid=64, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, UNASSIGN 2024-11-09T17:26:41,437 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=65 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:41,439 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:26:41,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; CloseRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:41,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:41,591 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:41,592 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(124): Close 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:41,592 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:26:41,592 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1681): Closing 9b0fa22acef654074d43b8d7d6d68c6a, disabling compactions & flushes 2024-11-09T17:26:41,592 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:41,592 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:41,592 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. after waiting 0 ms 2024-11-09T17:26:41,593 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:41,593 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(2837): Flushing 9b0fa22acef654074d43b8d7d6d68c6a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-09T17:26:41,593 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=A 2024-11-09T17:26:41,593 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:41,594 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=B 2024-11-09T17:26:41,594 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:41,594 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 9b0fa22acef654074d43b8d7d6d68c6a, store=C 2024-11-09T17:26:41,594 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:41,606 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109234bade8a18440be960dcc7d4910873d_9b0fa22acef654074d43b8d7d6d68c6a is 50, key is test_row_0/A:col10/1731173198486/Put/seqid=0 2024-11-09T17:26:41,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742035_1211 (size=12454) 2024-11-09T17:26:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:42,012 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:42,022 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109234bade8a18440be960dcc7d4910873d_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109234bade8a18440be960dcc7d4910873d_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:42,024 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/8cf48711c2454c04a970493395e28d6a, store: [table=TestAcidGuarantees family=A region=9b0fa22acef654074d43b8d7d6d68c6a] 2024-11-09T17:26:42,024 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/8cf48711c2454c04a970493395e28d6a is 175, key is test_row_0/A:col10/1731173198486/Put/seqid=0 2024-11-09T17:26:42,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742036_1212 (size=31255) 2024-11-09T17:26:42,431 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/8cf48711c2454c04a970493395e28d6a 2024-11-09T17:26:42,447 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/cfa803494f0c4bcfa06d37c5886a986f is 50, key is test_row_0/B:col10/1731173198486/Put/seqid=0 2024-11-09T17:26:42,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742037_1213 (size=12301) 2024-11-09T17:26:42,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:42,854 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/cfa803494f0c4bcfa06d37c5886a986f 2024-11-09T17:26:42,868 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/aa56b6ea29b84d4ab69205f8002f55df is 50, key is test_row_0/C:col10/1731173198486/Put/seqid=0 2024-11-09T17:26:42,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742038_1214 (size=12301) 2024-11-09T17:26:43,274 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/aa56b6ea29b84d4ab69205f8002f55df 2024-11-09T17:26:43,306 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/A/8cf48711c2454c04a970493395e28d6a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/8cf48711c2454c04a970493395e28d6a 2024-11-09T17:26:43,315 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/8cf48711c2454c04a970493395e28d6a, entries=150, sequenceid=321, filesize=30.5 K 2024-11-09T17:26:43,316 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/B/cfa803494f0c4bcfa06d37c5886a986f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/cfa803494f0c4bcfa06d37c5886a986f 2024-11-09T17:26:43,320 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/cfa803494f0c4bcfa06d37c5886a986f, entries=150, sequenceid=321, filesize=12.0 K 2024-11-09T17:26:43,321 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/.tmp/C/aa56b6ea29b84d4ab69205f8002f55df as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/aa56b6ea29b84d4ab69205f8002f55df 2024-11-09T17:26:43,326 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/aa56b6ea29b84d4ab69205f8002f55df, entries=150, sequenceid=321, filesize=12.0 K 2024-11-09T17:26:43,327 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(3040): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 9b0fa22acef654074d43b8d7d6d68c6a in 1734ms, sequenceid=321, compaction requested=true 2024-11-09T17:26:43,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e] to archive 2024-11-09T17:26:43,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:43,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/cf3be8e90a26435399110dc944a29455 2024-11-09T17:26:43,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f62386d323c6402daf88bc6aa87f34e3 2024-11-09T17:26:43,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/a67854eaebdb496bb91778c8a97f0abb 2024-11-09T17:26:43,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/53ceb31afd88431abbcfc18c21743300 2024-11-09T17:26:43,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/f011e8970a2249f39bfcfcc19def0624 2024-11-09T17:26:43,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/dbd56c5c23024e9fa4ffd93b7cfeb427 2024-11-09T17:26:43,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/d67b6ea1f1de4d9eb7425013a0d49ffd 2024-11-09T17:26:43,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/ecdb89911d0a42d68b1680b20528a0cf 2024-11-09T17:26:43,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6537ab7f61e64942a6d6c14543ef89d1 2024-11-09T17:26:43,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/9f264372e5f34cf4b6faa7afe98e8384 2024-11-09T17:26:43,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/74dbac443e264acdad114db280bdf913 2024-11-09T17:26:43,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/efdb0e1236074a80bc8bc963fc1ed5fd 2024-11-09T17:26:43,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/6fb715c1ecad45f5bd2e7b566f7458dc 2024-11-09T17:26:43,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/5c4ed30afd7a45f59534b8033ad37cb8 2024-11-09T17:26:43,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/38c59810c2f94461bdc5fe5ee82dcb61 2024-11-09T17:26:43,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b62067e48dc245f4a7699d2b3da16726 2024-11-09T17:26:43,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/b3024d1468454e18b2a131dd3705d0a3 2024-11-09T17:26:43,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/4ac62744b83c40a9812a694a54dac931 2024-11-09T17:26:43,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e01337d40d8144458411d22002c6ba36 2024-11-09T17:26:43,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/e2303363b89c4cc29434f1fd233f799e 2024-11-09T17:26:43,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bbe92a3e07c44b6a9ba791fb78a42b61, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8e83e3cd7a9e48c182360dba27a3a433, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/a395f232c7334547950eb6f9768b0724, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/61b933892b644f4987b6c4d800a0e611, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/798b4bbaf0ad42a39c4b686805488c3f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da] to archive 2024-11-09T17:26:43,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:43,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/3f7e5443a009496994bc7b5387dba600 2024-11-09T17:26:43,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/2000af22c567471e8f9686c535abab5f 2024-11-09T17:26:43,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bbe92a3e07c44b6a9ba791fb78a42b61 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bbe92a3e07c44b6a9ba791fb78a42b61 2024-11-09T17:26:43,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/aba1431ae1834ce184b83b92a309e9db 2024-11-09T17:26:43,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e8d271f9412c42b0a0388a2caedee08a 2024-11-09T17:26:43,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8e83e3cd7a9e48c182360dba27a3a433 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8e83e3cd7a9e48c182360dba27a3a433 2024-11-09T17:26:43,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f4cdeb355ec54fa5919fe0e0cc830951 2024-11-09T17:26:43,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/e14458b185294d30afe75d45bffa0a98 2024-11-09T17:26:43,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/ba240b1966b048b0a4ae9b9711c13362 2024-11-09T17:26:43,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/f13998adee7c47878a5407354ebaaf47 2024-11-09T17:26:43,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/a395f232c7334547950eb6f9768b0724 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/a395f232c7334547950eb6f9768b0724 2024-11-09T17:26:43,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/21af2fcd79ba4eca9462fafce5a74650 2024-11-09T17:26:43,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/05d79568b5cf43228fcea1db55cfef30 2024-11-09T17:26:43,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/61b933892b644f4987b6c4d800a0e611 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/61b933892b644f4987b6c4d800a0e611 2024-11-09T17:26:43,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/d885da92228f4dbfbffe71cd02628eee 2024-11-09T17:26:43,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/8b865fd9a79c4dc18bf0f36574f83a96 2024-11-09T17:26:43,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/798b4bbaf0ad42a39c4b686805488c3f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/798b4bbaf0ad42a39c4b686805488c3f 2024-11-09T17:26:43,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/bc45a2f236d1412aa911bc1df1777657 2024-11-09T17:26:43,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/95eeb3b1e32247348dd217c9d7706d40 2024-11-09T17:26:43,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/726164f748a24358b69ace305f7cc7da 2024-11-09T17:26:43,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/ee8e802969fd4afb8bc9522901e048e3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/8bb93896b9b94c2dbde492d8df1cf546, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/adedcecd764f479cb639e9a38bfe2f94, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/1606febc243641a9a2344f1216b07b8e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/a51b830b109f4901be5d8ce2ed7e8bb7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59] to archive 2024-11-09T17:26:43,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:26:43,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44b42c31cc4942b68759b15ee588ba7b 2024-11-09T17:26:43,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f5fb89252c894114aab8e5795f66fbd2 2024-11-09T17:26:43,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/ee8e802969fd4afb8bc9522901e048e3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/ee8e802969fd4afb8bc9522901e048e3 2024-11-09T17:26:43,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/6ec7bdbd4ea140dcbf934b1a81f70a28 2024-11-09T17:26:43,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e6e66adc88db486fb0ebfa237d1a3224 2024-11-09T17:26:43,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/8bb93896b9b94c2dbde492d8df1cf546 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/8bb93896b9b94c2dbde492d8df1cf546 2024-11-09T17:26:43,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/2efbff6657b94c688b75e1da7824921d 2024-11-09T17:26:43,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5c78bf98a73b4ebcb71790b1844ff10f 2024-11-09T17:26:43,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/e4e02ae23e2a4f8b8e3facc57f994f8a 2024-11-09T17:26:43,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/44ca469d175e4303addaea59983a1c21 2024-11-09T17:26:43,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/adedcecd764f479cb639e9a38bfe2f94 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/adedcecd764f479cb639e9a38bfe2f94 2024-11-09T17:26:43,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/89645a1dc912491b8408f05f9f4d2d9f 2024-11-09T17:26:43,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/f1f8d58f66224a3984178ac3ba83a789 2024-11-09T17:26:43,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/1606febc243641a9a2344f1216b07b8e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/1606febc243641a9a2344f1216b07b8e 2024-11-09T17:26:43,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/297307066c994239bb9605af385b44ee 2024-11-09T17:26:43,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/b8c5bf4e15004a5a9e0e65245b8f5415 2024-11-09T17:26:43,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/a51b830b109f4901be5d8ce2ed7e8bb7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/a51b830b109f4901be5d8ce2ed7e8bb7 2024-11-09T17:26:43,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/821c8339edf2424fba9a7d5db2bc6ab6 2024-11-09T17:26:43,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/420f6eed05054db3aa4df0aa48078ab5 2024-11-09T17:26:43,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/0cc907bc0c654863a996e532de6b5a59 2024-11-09T17:26:43,391 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-09T17:26:43,391 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a. 2024-11-09T17:26:43,392 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] regionserver.HRegion(1635): Region close journal for 9b0fa22acef654074d43b8d7d6d68c6a: 2024-11-09T17:26:43,393 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=66}] handler.UnassignRegionHandler(170): Closed 9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,393 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=65 updating hbase:meta row=9b0fa22acef654074d43b8d7d6d68c6a, regionState=CLOSED 2024-11-09T17:26:43,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-11-09T17:26:43,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; CloseRegionProcedure 9b0fa22acef654074d43b8d7d6d68c6a, server=c28fd47d982f,45633,1731173138129 in 1.9550 sec 2024-11-09T17:26:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T17:26:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, ppid=64, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=9b0fa22acef654074d43b8d7d6d68c6a, UNASSIGN in 1.9610 sec 2024-11-09T17:26:43,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-09T17:26:43,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9650 sec 2024-11-09T17:26:43,397 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173203397"}]},"ts":"1731173203397"} 2024-11-09T17:26:43,398 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:26:43,411 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:26:43,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 2.0340 sec 2024-11-09T17:26:43,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-09T17:26:43,487 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 63 completed 2024-11-09T17:26:43,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:26:43,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,490 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=67, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-09T17:26:43,490 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=67, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,493 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,497 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits] 2024-11-09T17:26:43,501 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/7580ab72ebae4029a4633335f9b75154 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/7580ab72ebae4029a4633335f9b75154 2024-11-09T17:26:43,503 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/8cf48711c2454c04a970493395e28d6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/8cf48711c2454c04a970493395e28d6a 2024-11-09T17:26:43,504 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/95cb5d8c277a4983aef81451d744d511 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/A/95cb5d8c277a4983aef81451d744d511 2024-11-09T17:26:43,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7ca85e11d86246b2a97ef349b5e6d38c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7ca85e11d86246b2a97ef349b5e6d38c 2024-11-09T17:26:43,508 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7fb78d4a73d442c48365792849d87cd2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/7fb78d4a73d442c48365792849d87cd2 2024-11-09T17:26:43,509 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/cfa803494f0c4bcfa06d37c5886a986f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/B/cfa803494f0c4bcfa06d37c5886a986f 2024-11-09T17:26:43,512 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5ad0fc8009934bbe842052975d439637 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/5ad0fc8009934bbe842052975d439637 2024-11-09T17:26:43,513 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/aa56b6ea29b84d4ab69205f8002f55df to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/aa56b6ea29b84d4ab69205f8002f55df 2024-11-09T17:26:43,514 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/c21041ddddb24f51892f1288a1d26698 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/C/c21041ddddb24f51892f1288a1d26698 2024-11-09T17:26:43,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits/324.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a/recovered.edits/324.seqid 2024-11-09T17:26:43,516 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,516 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:26:43,517 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:26:43,518 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T17:26:43,521 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110913cd058087344fe1a2bfc5dedd85d513_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110913cd058087344fe1a2bfc5dedd85d513_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,523 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109150bde2b09d34f208b65e446932d72fc_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109150bde2b09d34f208b65e446932d72fc_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,524 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091dcef657b30543d4b0bbc00fcc434073_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091dcef657b30543d4b0bbc00fcc434073_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,525 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e98c5aad43347a8a47024d19abddbe3_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091e98c5aad43347a8a47024d19abddbe3_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,526 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109234bade8a18440be960dcc7d4910873d_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109234bade8a18440be960dcc7d4910873d_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,527 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925b35bcb07d347d7b3789b91ca83319e_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925b35bcb07d347d7b3789b91ca83319e_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,528 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092d8d70eaef3746f597005076503aabfc_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092d8d70eaef3746f597005076503aabfc_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,529 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110937b21b021b064bebabfab86873185c84_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110937b21b021b064bebabfab86873185c84_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,530 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094030f53fc8dc4a4e8f87783523aed765_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094030f53fc8dc4a4e8f87783523aed765_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,531 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109571ad924dc6b4a7dbc5a992aa3e959a9_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109571ad924dc6b4a7dbc5a992aa3e959a9_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,532 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c3e11dbc51f4510af5018e783468bcb_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095c3e11dbc51f4510af5018e783468bcb_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,533 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099980ae6a58204ca5857da60ea1312bc4_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099980ae6a58204ca5857da60ea1312bc4_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,534 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b53777e3f6c745bd908d5959dcabb602_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b53777e3f6c745bd908d5959dcabb602_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,535 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b612a760d8ec438b974d686592fc8f6b_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b612a760d8ec438b974d686592fc8f6b_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,535 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba1cd9808c0a4f66a0289088aae6969e_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ba1cd9808c0a4f66a0289088aae6969e_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,537 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ea5b8c834f56405aa4c7efb50132eac5_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ea5b8c834f56405aa4c7efb50132eac5_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f14ed9b3bdf940018bba1295f58e5f01_9b0fa22acef654074d43b8d7d6d68c6a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f14ed9b3bdf940018bba1295f58e5f01_9b0fa22acef654074d43b8d7d6d68c6a 2024-11-09T17:26:43,538 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:26:43,540 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=67, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,542 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:26:43,544 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:26:43,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=67, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,545 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:26:43,546 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173203545"}]},"ts":"9223372036854775807"} 2024-11-09T17:26:43,548 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:26:43,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => 9b0fa22acef654074d43b8d7d6d68c6a, NAME => 'TestAcidGuarantees,,1731173174065.9b0fa22acef654074d43b8d7d6d68c6a.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:26:43,548 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:26:43,548 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173203548"}]},"ts":"9223372036854775807"} 2024-11-09T17:26:43,550 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:26:43,555 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=67, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 67 msec 2024-11-09T17:26:43,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-09T17:26:43,591 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 67 completed 2024-11-09T17:26:43,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-195685394_22 at /127.0.0.1:48838 [Waiting for operation #353] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-195685394_22 at /127.0.0.1:48688 [Waiting for operation #392] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-11 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-898056791_22 at /127.0.0.1:44192 [Waiting for operation #418] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-898056791_22 at /127.0.0.1:48854 [Waiting for operation #328] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hconnection-0x4a2c6199-shared-pool-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=465 (was 456) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=298 (was 281) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5936 (was 6066) 2024-11-09T17:26:43,608 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testGetAtomicity Thread=242, OpenFileDescriptor=465, MaxFileDescriptor=1048576, SystemLoadAverage=298, ProcessCount=11, AvailableMemoryMB=5936 2024-11-09T17:26:43,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:26:43,610 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:26:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=68, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:26:43,611 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:26:43,611 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:43,612 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 68 2024-11-09T17:26:43,612 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:26:43,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:43,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742039_1215 (size=960) 2024-11-09T17:26:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:43,985 DEBUG [master/c28fd47d982f:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 8909151a42b30d353536c39084e3c0b3 changed from -1.0 to 0.0, refreshing cache 2024-11-09T17:26:44,019 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:26:44,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742040_1216 (size=53) 2024-11-09T17:26:44,075 ERROR [LeaseRenewer:jenkins@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46491,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:44,428 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:26:44,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing bbbe0e290bd0d6582b636c1e26d5e726, disabling compactions & flushes 2024-11-09T17:26:44,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. after waiting 0 ms 2024-11-09T17:26:44,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,429 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,429 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:44,432 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:26:44,432 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173204432"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173204432"}]},"ts":"1731173204432"} 2024-11-09T17:26:44,433 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:26:44,434 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:26:44,434 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173204434"}]},"ts":"1731173204434"} 2024-11-09T17:26:44,435 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:26:44,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, ASSIGN}] 2024-11-09T17:26:44,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, ASSIGN 2024-11-09T17:26:44,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=69, ppid=68, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:26:44,667 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bbbe0e290bd0d6582b636c1e26d5e726, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:44,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE; OpenRegionProcedure bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:26:44,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:44,822 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:44,829 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,830 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7285): Opening region: {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:26:44,831 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,831 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:26:44,831 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7327): checking encryption for bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,831 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(7330): checking classloading for bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,835 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,838 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:44,839 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbbe0e290bd0d6582b636c1e26d5e726 columnFamilyName A 2024-11-09T17:26:44,839 DEBUG [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:44,839 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(327): Store=bbbe0e290bd0d6582b636c1e26d5e726/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:44,839 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,841 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:44,841 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbbe0e290bd0d6582b636c1e26d5e726 columnFamilyName B 2024-11-09T17:26:44,841 DEBUG [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:44,842 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(327): Store=bbbe0e290bd0d6582b636c1e26d5e726/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:44,842 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,844 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:26:44,844 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbbe0e290bd0d6582b636c1e26d5e726 columnFamilyName C 2024-11-09T17:26:44,844 DEBUG [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:26:44,845 INFO [StoreOpener-bbbe0e290bd0d6582b636c1e26d5e726-1 {}] regionserver.HStore(327): Store=bbbe0e290bd0d6582b636c1e26d5e726/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:26:44,845 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,846 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,846 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,848 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:26:44,849 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1085): writing seq id for bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:44,851 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:26:44,851 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1102): Opened bbbe0e290bd0d6582b636c1e26d5e726; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66488570, jitterRate=-0.009243100881576538}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:26:44,852 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegion(1001): Region open journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:44,853 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., pid=70, masterSystemTime=1731173204822 2024-11-09T17:26:44,854 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,854 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=70}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:44,854 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=69 updating hbase:meta row=bbbe0e290bd0d6582b636c1e26d5e726, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:44,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=70, resume processing ppid=69 2024-11-09T17:26:44,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=70, ppid=69, state=SUCCESS; OpenRegionProcedure bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 in 186 msec 2024-11-09T17:26:44,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T17:26:44,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=69, ppid=68, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, ASSIGN in 343 msec 2024-11-09T17:26:44,857 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:26:44,858 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173204857"}]},"ts":"1731173204857"} 2024-11-09T17:26:44,858 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:26:44,863 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=68, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:26:44,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2530 sec 2024-11-09T17:26:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=68 2024-11-09T17:26:45,721 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 68 completed 2024-11-09T17:26:45,727 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a569490 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c1ac389 2024-11-09T17:26:45,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44645c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,777 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45176, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:26:45,781 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37840, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:26:45,787 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6862e3ce to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28e73c0 2024-11-09T17:26:45,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64ee0130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,799 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d296fed to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c480dfb 2024-11-09T17:26:45,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@683b64c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,808 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08d0caa5 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34cb3991 2024-11-09T17:26:45,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e55eb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,823 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43f04e0e to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e9ae050 2024-11-09T17:26:45,831 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a703d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,833 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x560ec309 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fef31f8 2024-11-09T17:26:45,846 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14ed1e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,849 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0eb04aeb to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72537a47 2024-11-09T17:26:45,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@88aa519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,865 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6a0e9c8f to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36642cb 2024-11-09T17:26:45,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e998dd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,881 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d68f787 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c299cfb 2024-11-09T17:26:45,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e4c79b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,891 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10e6bf6a to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@605827c9 2024-11-09T17:26:45,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1403c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,906 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1730a60f to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3677bd4f 2024-11-09T17:26:45,914 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bf0ba59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:26:45,919 DEBUG [hconnection-0x108109bf-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,919 DEBUG [hconnection-0xeaef610-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,921 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45182, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,921 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45180, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,922 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:45,923 DEBUG [hconnection-0x4a1dde49-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,923 DEBUG [hconnection-0x29d977d2-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,923 DEBUG [hconnection-0x54710e53-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,923 DEBUG [hconnection-0x71eee384-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,924 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45192, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,924 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45196, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,924 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45210, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees 2024-11-09T17:26:45,925 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=71, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:45,925 DEBUG [hconnection-0x5aba5d0a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,925 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45214, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:45,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=71, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:45,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:45,928 DEBUG [hconnection-0x2fa6ce6a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,928 DEBUG [hconnection-0x3cd2356b-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,929 DEBUG [hconnection-0x627ffc69-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:26:45,934 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45216, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,934 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45230, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,934 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45242, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,934 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45246, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:26:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:45,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:45,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:46,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/70b178e811da468eb2bc669017dc3dbb is 50, key is test_row_0/A:col10/1731173205979/Put/seqid=0 2024-11-09T17:26:46,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742041_1217 (size=12001) 2024-11-09T17:26:46,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173266043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173266043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173266047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173266047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173266047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,079 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173266148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173266151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173266151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173266151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:46,232 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173266351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173266351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173266354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173266354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173266355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,385 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/70b178e811da468eb2bc669017dc3dbb 2024-11-09T17:26:46,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/c0c1a672d6e7421ca025892b75c544d4 is 50, key is test_row_0/B:col10/1731173205979/Put/seqid=0 2024-11-09T17:26:46,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742042_1218 (size=12001) 2024-11-09T17:26:46,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/c0c1a672d6e7421ca025892b75c544d4 2024-11-09T17:26:46,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cef592a1fa4a4f76b3443b22124b2af3 is 50, key is test_row_0/C:col10/1731173205979/Put/seqid=0 2024-11-09T17:26:46,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742043_1219 (size=12001) 2024-11-09T17:26:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:46,538 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173266654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173266654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173266657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173266657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:46,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173266657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,691 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,843 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] handler.RSProcedureHandler(58): pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=72 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=72 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:46,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cef592a1fa4a4f76b3443b22124b2af3 2024-11-09T17:26:46,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/70b178e811da468eb2bc669017dc3dbb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb 2024-11-09T17:26:46,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T17:26:46,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/c0c1a672d6e7421ca025892b75c544d4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4 2024-11-09T17:26:46,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T17:26:46,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cef592a1fa4a4f76b3443b22124b2af3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3 2024-11-09T17:26:46,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T17:26:46,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for bbbe0e290bd0d6582b636c1e26d5e726 in 906ms, sequenceid=14, compaction requested=false 2024-11-09T17:26:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:46,995 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:46,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=72 2024-11-09T17:26:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:46,996 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:26:46,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:46,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:47,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7cfe2979e6344a99a005082b9ecde6d6 is 50, key is test_row_0/A:col10/1731173206045/Put/seqid=0 2024-11-09T17:26:47,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742044_1220 (size=12001) 2024-11-09T17:26:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:47,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:47,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:47,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173267164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173267167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173267168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173267168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173267168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173267269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173267271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173267271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173267271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173267272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,408 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7cfe2979e6344a99a005082b9ecde6d6 2024-11-09T17:26:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/77080da8ce5a4021a9d93f4d998b5bdb is 50, key is test_row_0/B:col10/1731173206045/Put/seqid=0 2024-11-09T17:26:47,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742045_1221 (size=12001) 2024-11-09T17:26:47,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173267471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173267473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173267474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173267474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173267474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173267775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173267776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173267776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173267777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173267778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:47,819 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/77080da8ce5a4021a9d93f4d998b5bdb 2024-11-09T17:26:47,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d4c24008132249bdafe7ea7a3a17fa69 is 50, key is test_row_0/C:col10/1731173206045/Put/seqid=0 2024-11-09T17:26:47,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742046_1222 (size=12001) 2024-11-09T17:26:47,833 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d4c24008132249bdafe7ea7a3a17fa69 2024-11-09T17:26:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7cfe2979e6344a99a005082b9ecde6d6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6 2024-11-09T17:26:47,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T17:26:47,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/77080da8ce5a4021a9d93f4d998b5bdb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb 2024-11-09T17:26:47,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T17:26:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d4c24008132249bdafe7ea7a3a17fa69 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69 2024-11-09T17:26:47,850 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69, entries=150, sequenceid=39, filesize=11.7 K 2024-11-09T17:26:47,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for bbbe0e290bd0d6582b636c1e26d5e726 in 854ms, sequenceid=39, compaction requested=false 2024-11-09T17:26:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:47,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-11-09T17:26:47,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-11-09T17:26:47,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-11-09T17:26:47,853 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-11-09T17:26:47,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=71, table=TestAcidGuarantees in 1.9320 sec 2024-11-09T17:26:48,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-09T17:26:48,029 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 71 completed 2024-11-09T17:26:48,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees 2024-11-09T17:26:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-09T17:26:48,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=73, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:48,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=73, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:48,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-09T17:26:48,183 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=74 2024-11-09T17:26:48,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:48,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:48,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/32b580b169e242edaef7aec352faf91d is 50, key is test_row_0/A:col10/1731173207166/Put/seqid=0 2024-11-09T17:26:48,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742047_1223 (size=9657) 2024-11-09T17:26:48,193 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/32b580b169e242edaef7aec352faf91d 2024-11-09T17:26:48,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/f65112cb04e54b1ebaad383683b3ec2a is 50, key is test_row_0/B:col10/1731173207166/Put/seqid=0 2024-11-09T17:26:48,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742048_1224 (size=9657) 2024-11-09T17:26:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173268323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173268325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173268325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173268326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173268327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-09T17:26:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173268428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173268428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173268428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173268429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173268430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,604 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/f65112cb04e54b1ebaad383683b3ec2a 2024-11-09T17:26:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af is 50, key is test_row_0/C:col10/1731173207166/Put/seqid=0 2024-11-09T17:26:48,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742049_1225 (size=9657) 2024-11-09T17:26:48,627 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af 2024-11-09T17:26:48,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/32b580b169e242edaef7aec352faf91d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d 2024-11-09T17:26:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-09T17:26:48,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173268630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173268631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173268631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173268632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,637 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d, entries=100, sequenceid=51, filesize=9.4 K 2024-11-09T17:26:48,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/f65112cb04e54b1ebaad383683b3ec2a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a 2024-11-09T17:26:48,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173268635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a, entries=100, sequenceid=51, filesize=9.4 K 2024-11-09T17:26:48,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af 2024-11-09T17:26:48,650 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af, entries=100, sequenceid=51, filesize=9.4 K 2024-11-09T17:26:48,652 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bbbe0e290bd0d6582b636c1e26d5e726 in 469ms, sequenceid=51, compaction requested=true 2024-11-09T17:26:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:48,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-11-09T17:26:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-11-09T17:26:48,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-09T17:26:48,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-09T17:26:48,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=73, table=TestAcidGuarantees in 625 msec 2024-11-09T17:26:48,920 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:26:48,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:26:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:48,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:48,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:48,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:48,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:48,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/01fae30aeb4a4648a180e0645566d716 is 50, key is test_row_0/A:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173268944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173268944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173268945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173268947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173268949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742050_1226 (size=14341) 2024-11-09T17:26:49,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173269049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173269049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173269050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173269052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173269052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-09T17:26:49,135 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 73 completed 2024-11-09T17:26:49,136 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees 2024-11-09T17:26:49,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:49,137 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:49,138 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=75, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:49,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173269252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173269252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173269254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173269255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173269255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,289 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:49,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:49,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/01fae30aeb4a4648a180e0645566d716 2024-11-09T17:26:49,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/2d43b569892d4a0aa0522325fbf93964 is 50, key is test_row_0/B:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:49,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742051_1227 (size=12001) 2024-11-09T17:26:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:49,442 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:49,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173269554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173269557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173269557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173269557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173269558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,594 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:49,747 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:49,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/2d43b569892d4a0aa0522325fbf93964 2024-11-09T17:26:49,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/f3466964c07c4f16ab269b1c97a7b73c is 50, key is test_row_0/C:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:49,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742052_1228 (size=12001) 2024-11-09T17:26:49,899 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:49,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:49,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:50,051 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] handler.RSProcedureHandler(58): pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:50,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=76 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=76 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:50,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173270059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173270059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173270060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173270062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173270064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/f3466964c07c4f16ab269b1c97a7b73c 2024-11-09T17:26:50,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/01fae30aeb4a4648a180e0645566d716 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716 2024-11-09T17:26:50,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716, entries=200, sequenceid=77, filesize=14.0 K 2024-11-09T17:26:50,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/2d43b569892d4a0aa0522325fbf93964 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964 2024-11-09T17:26:50,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T17:26:50,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/f3466964c07c4f16ab269b1c97a7b73c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c 2024-11-09T17:26:50,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T17:26:50,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bbbe0e290bd0d6582b636c1e26d5e726 in 1258ms, sequenceid=77, compaction requested=true 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:50,195 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:50,195 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48000 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:26:50,197 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,197 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,197 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=46.9 K 2024-11-09T17:26:50,197 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=44.6 K 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 70b178e811da468eb2bc669017dc3dbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731173205923 2024-11-09T17:26:50,197 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c0c1a672d6e7421ca025892b75c544d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731173205923 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7cfe2979e6344a99a005082b9ecde6d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731173206045 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 77080da8ce5a4021a9d93f4d998b5bdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731173206045 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 32b580b169e242edaef7aec352faf91d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731173207166 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f65112cb04e54b1ebaad383683b3ec2a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731173207166 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 01fae30aeb4a4648a180e0645566d716, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208324 2024-11-09T17:26:50,198 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2d43b569892d4a0aa0522325fbf93964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208326 2024-11-09T17:26:50,204 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:50,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=76 2024-11-09T17:26:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T17:26:50,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:50,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:50,208 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:50,209 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/088091985d7549688b01f2ea3cfe4d74 is 50, key is test_row_0/A:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:50,209 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:50,210 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/63dc6879e07c47839bb628a8bd117b3b is 50, key is test_row_0/B:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/526834907d3844d4b6bf2ebf3613de9d is 50, key is test_row_0/A:col10/1731173208945/Put/seqid=0 2024-11-09T17:26:50,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742054_1230 (size=12139) 2024-11-09T17:26:50,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742053_1229 (size=12139) 2024-11-09T17:26:50,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742055_1231 (size=12001) 2024-11-09T17:26:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:50,631 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/526834907d3844d4b6bf2ebf3613de9d 2024-11-09T17:26:50,635 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/088091985d7549688b01f2ea3cfe4d74 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/088091985d7549688b01f2ea3cfe4d74 2024-11-09T17:26:50,636 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/63dc6879e07c47839bb628a8bd117b3b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/63dc6879e07c47839bb628a8bd117b3b 2024-11-09T17:26:50,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/56d280e264bc49a2854b2fb201c13330 is 50, key is test_row_0/B:col10/1731173208945/Put/seqid=0 2024-11-09T17:26:50,641 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 088091985d7549688b01f2ea3cfe4d74(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:50,641 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=12, startTime=1731173210195; duration=0sec 2024-11-09T17:26:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:26:50,641 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:50,642 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into 63dc6879e07c47839bb628a8bd117b3b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:50,642 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:50,642 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=12, startTime=1731173210195; duration=0sec 2024-11-09T17:26:50,642 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:50,642 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:26:50,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742056_1232 (size=12001) 2024-11-09T17:26:50,644 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:50,644 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:26:50,644 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:50,644 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=44.6 K 2024-11-09T17:26:50,645 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cef592a1fa4a4f76b3443b22124b2af3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731173205923 2024-11-09T17:26:50,645 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d4c24008132249bdafe7ea7a3a17fa69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731173206045 2024-11-09T17:26:50,645 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3bdd4bfe53e14ec8a63dd1fdb2a9e2af, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731173207166 2024-11-09T17:26:50,645 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f3466964c07c4f16ab269b1c97a7b73c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208326 2024-11-09T17:26:50,676 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#190 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:50,677 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/2187dac00a0c431fae75156201283392 is 50, key is test_row_0/C:col10/1731173208326/Put/seqid=0 2024-11-09T17:26:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742057_1233 (size=12139) 2024-11-09T17:26:50,692 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/2187dac00a0c431fae75156201283392 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2187dac00a0c431fae75156201283392 2024-11-09T17:26:50,697 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into 2187dac00a0c431fae75156201283392(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:50,697 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:50,697 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=12, startTime=1731173210195; duration=0sec 2024-11-09T17:26:50,697 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:50,697 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:26:51,044 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/56d280e264bc49a2854b2fb201c13330 2024-11-09T17:26:51,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/c4c710ae2b65475ead483bba0c622120 is 50, key is test_row_0/C:col10/1731173208945/Put/seqid=0 2024-11-09T17:26:51,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742058_1234 (size=12001) 2024-11-09T17:26:51,061 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/c4c710ae2b65475ead483bba0c622120 2024-11-09T17:26:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:51,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:51,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/526834907d3844d4b6bf2ebf3613de9d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d 2024-11-09T17:26:51,071 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d, entries=150, sequenceid=87, filesize=11.7 K 2024-11-09T17:26:51,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/56d280e264bc49a2854b2fb201c13330 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330 2024-11-09T17:26:51,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330, entries=150, sequenceid=87, filesize=11.7 K 2024-11-09T17:26:51,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/c4c710ae2b65475ead483bba0c622120 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120 2024-11-09T17:26:51,087 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120, entries=150, sequenceid=87, filesize=11.7 K 2024-11-09T17:26:51,112 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=120.76 KB/123660 for bbbe0e290bd0d6582b636c1e26d5e726 in 908ms, sequenceid=87, compaction requested=false 2024-11-09T17:26:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:51,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-11-09T17:26:51,113 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:26:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:51,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:51,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-09T17:26:51,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-09T17:26:51,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/2338bdbb82e246c0a66f4f97791b90cf is 50, key is test_row_0/A:col10/1731173211082/Put/seqid=0 2024-11-09T17:26:51,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=75, table=TestAcidGuarantees in 1.9790 sec 2024-11-09T17:26:51,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742059_1235 (size=16681) 2024-11-09T17:26:51,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173271124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173271125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173271125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173271126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173271126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173271229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173271230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-09T17:26:51,240 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 75 completed 2024-11-09T17:26:51,241 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees 2024-11-09T17:26:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-09T17:26:51,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:51,243 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=77, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:51,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-09T17:26:51,394 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-09T17:26:51,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:51,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173271432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173271432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173271433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173271433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173271435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/2338bdbb82e246c0a66f4f97791b90cf 2024-11-09T17:26:51,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d8ae96863df341d1843c5e14f3384433 is 50, key is test_row_0/B:col10/1731173211082/Put/seqid=0 2024-11-09T17:26:51,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742060_1236 (size=12001) 2024-11-09T17:26:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-09T17:26:51,546 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-09T17:26:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,699 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-09T17:26:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173271735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173271736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173271737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173271737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173271738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-09T17:26:51,851 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:51,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-09T17:26:51,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] handler.RSProcedureHandler(58): pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=78 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=78 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:51,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d8ae96863df341d1843c5e14f3384433 2024-11-09T17:26:51,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/268c9c74d1d547bab8c3a85e2d81029c is 50, key is test_row_0/C:col10/1731173211082/Put/seqid=0 2024-11-09T17:26:51,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742061_1237 (size=12001) 2024-11-09T17:26:51,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/268c9c74d1d547bab8c3a85e2d81029c 2024-11-09T17:26:51,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/2338bdbb82e246c0a66f4f97791b90cf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf 2024-11-09T17:26:51,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf, entries=250, sequenceid=114, filesize=16.3 K 2024-11-09T17:26:51,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d8ae96863df341d1843c5e14f3384433 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433 2024-11-09T17:26:51,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T17:26:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/268c9c74d1d547bab8c3a85e2d81029c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c 2024-11-09T17:26:51,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c, entries=150, sequenceid=114, filesize=11.7 K 2024-11-09T17:26:51,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for bbbe0e290bd0d6582b636c1e26d5e726 in 856ms, sequenceid=114, compaction requested=true 2024-11-09T17:26:51,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:51,968 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:51,969 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:51,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:26:51,970 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,970 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:51,970 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/63dc6879e07c47839bb628a8bd117b3b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=35.3 K 2024-11-09T17:26:51,970 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/088091985d7549688b01f2ea3cfe4d74, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=39.9 K 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 088091985d7549688b01f2ea3cfe4d74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208326 2024-11-09T17:26:51,970 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 63dc6879e07c47839bb628a8bd117b3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208326 2024-11-09T17:26:51,971 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 56d280e264bc49a2854b2fb201c13330, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731173208940 2024-11-09T17:26:51,971 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 526834907d3844d4b6bf2ebf3613de9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731173208940 2024-11-09T17:26:51,971 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2338bdbb82e246c0a66f4f97791b90cf, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731173211078 2024-11-09T17:26:51,971 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d8ae96863df341d1843c5e14f3384433, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731173211078 2024-11-09T17:26:51,978 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:51,979 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d2dea898a5914d17acefe7d2e2d12e37 is 50, key is test_row_0/B:col10/1731173211082/Put/seqid=0 2024-11-09T17:26:51,980 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:51,980 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/70540dea243e49c19dc97f2fcacb5d5c is 50, key is test_row_0/A:col10/1731173211082/Put/seqid=0 2024-11-09T17:26:51,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742062_1238 (size=12241) 2024-11-09T17:26:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742063_1239 (size=12241) 2024-11-09T17:26:52,003 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=78 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,004 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:52,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e0a55c5df6a04015ad1b2e8050f5d4c9 is 50, key is test_row_0/A:col10/1731173211125/Put/seqid=0 2024-11-09T17:26:52,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742064_1240 (size=12001) 2024-11-09T17:26:52,024 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e0a55c5df6a04015ad1b2e8050f5d4c9 2024-11-09T17:26:52,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/25d2348a1b8f4b45b7abd15e5999b23f is 50, key is test_row_0/B:col10/1731173211125/Put/seqid=0 2024-11-09T17:26:52,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742065_1241 (size=12001) 2024-11-09T17:26:52,041 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/25d2348a1b8f4b45b7abd15e5999b23f 2024-11-09T17:26:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/60b52dcf6f8c45a68fc1bd2611011c8f is 50, key is test_row_0/C:col10/1731173211125/Put/seqid=0 2024-11-09T17:26:52,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742066_1242 (size=12001) 2024-11-09T17:26:52,064 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/60b52dcf6f8c45a68fc1bd2611011c8f 2024-11-09T17:26:52,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e0a55c5df6a04015ad1b2e8050f5d4c9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9 2024-11-09T17:26:52,072 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T17:26:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/25d2348a1b8f4b45b7abd15e5999b23f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f 2024-11-09T17:26:52,077 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T17:26:52,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/60b52dcf6f8c45a68fc1bd2611011c8f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f 2024-11-09T17:26:52,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f, entries=150, sequenceid=128, filesize=11.7 K 2024-11-09T17:26:52,082 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for bbbe0e290bd0d6582b636c1e26d5e726 in 78ms, sequenceid=128, compaction requested=true 2024-11-09T17:26:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-11-09T17:26:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-11-09T17:26:52,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-11-09T17:26:52,085 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-11-09T17:26:52,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=77, table=TestAcidGuarantees in 844 msec 2024-11-09T17:26:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:52,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:52,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:52,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7503d2c583d943af84cdb226d25e8b6c is 50, key is test_row_0/A:col10/1731173212246/Put/seqid=0 2024-11-09T17:26:52,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742067_1243 (size=14541) 2024-11-09T17:26:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173272264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173272265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173272267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173272267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173272268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-09T17:26:52,345 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 77 completed 2024-11-09T17:26:52,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees 2024-11-09T17:26:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:52,347 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:52,348 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=79, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:52,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:52,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173272369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173272369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173272371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173272374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173272374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,387 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d2dea898a5914d17acefe7d2e2d12e37 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d2dea898a5914d17acefe7d2e2d12e37 2024-11-09T17:26:52,388 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/70540dea243e49c19dc97f2fcacb5d5c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70540dea243e49c19dc97f2fcacb5d5c 2024-11-09T17:26:52,392 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into d2dea898a5914d17acefe7d2e2d12e37(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:52,392 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173211969; duration=0sec 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:52,392 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 70540dea243e49c19dc97f2fcacb5d5c(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:52,392 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173211968; duration=0sec 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:52,392 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:26:52,393 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:52,393 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:26:52,394 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,394 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2187dac00a0c431fae75156201283392, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=47.0 K 2024-11-09T17:26:52,394 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2187dac00a0c431fae75156201283392, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173208326 2024-11-09T17:26:52,394 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c4c710ae2b65475ead483bba0c622120, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731173208940 2024-11-09T17:26:52,395 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 268c9c74d1d547bab8c3a85e2d81029c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731173211078 2024-11-09T17:26:52,395 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 60b52dcf6f8c45a68fc1bd2611011c8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731173211118 2024-11-09T17:26:52,402 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:52,402 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cf4b8e1933214feeb9307e33926504c4 is 50, key is test_row_0/C:col10/1731173211125/Put/seqid=0 2024-11-09T17:26:52,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742068_1244 (size=12275) 2024-11-09T17:26:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:52,499 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173272572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173272572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173272574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173272577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173272578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:52,652 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7503d2c583d943af84cdb226d25e8b6c 2024-11-09T17:26:52,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/52ade52a99c249b485089af7885c1fa4 is 50, key is test_row_0/B:col10/1731173212246/Put/seqid=0 2024-11-09T17:26:52,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742069_1245 (size=12151) 2024-11-09T17:26:52,804 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:52,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:52,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,816 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cf4b8e1933214feeb9307e33926504c4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cf4b8e1933214feeb9307e33926504c4 2024-11-09T17:26:52,820 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into cf4b8e1933214feeb9307e33926504c4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:52,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:52,820 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=12, startTime=1731173211969; duration=0sec 2024-11-09T17:26:52,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:52,820 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:26:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173272876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173272878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173272879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173272880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173272880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:52,956 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:52,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:52,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:52,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:52,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/52ade52a99c249b485089af7885c1fa4 2024-11-09T17:26:53,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/50fb3b21de7c40088385ad292afe7ed1 is 50, key is test_row_0/C:col10/1731173212246/Put/seqid=0 2024-11-09T17:26:53,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742070_1246 (size=12151) 2024-11-09T17:26:53,108 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,260 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:53,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173273381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:53,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173273385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,413 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] handler.RSProcedureHandler(58): pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=80 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=80 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:53,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:53,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/50fb3b21de7c40088385ad292afe7ed1 2024-11-09T17:26:53,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7503d2c583d943af84cdb226d25e8b6c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c 2024-11-09T17:26:53,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c, entries=200, sequenceid=141, filesize=14.2 K 2024-11-09T17:26:53,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/52ade52a99c249b485089af7885c1fa4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4 2024-11-09T17:26:53,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4, entries=150, sequenceid=141, filesize=11.9 K 2024-11-09T17:26:53,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/50fb3b21de7c40088385ad292afe7ed1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1 2024-11-09T17:26:53,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1, entries=150, sequenceid=141, filesize=11.9 K 2024-11-09T17:26:53,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bbbe0e290bd0d6582b636c1e26d5e726 in 1266ms, sequenceid=141, compaction requested=true 2024-11-09T17:26:53,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:53,514 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:53,514 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:53,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:26:53,515 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,515 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,515 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70540dea243e49c19dc97f2fcacb5d5c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=37.9 K 2024-11-09T17:26:53,515 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d2dea898a5914d17acefe7d2e2d12e37, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=35.5 K 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d2dea898a5914d17acefe7d2e2d12e37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731173211078 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 70540dea243e49c19dc97f2fcacb5d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731173211078 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e0a55c5df6a04015ad1b2e8050f5d4c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731173211118 2024-11-09T17:26:53,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 25d2348a1b8f4b45b7abd15e5999b23f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731173211118 2024-11-09T17:26:53,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7503d2c583d943af84cdb226d25e8b6c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731173212243 2024-11-09T17:26:53,516 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 52ade52a99c249b485089af7885c1fa4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731173212246 2024-11-09T17:26:53,523 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:53,523 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:53,523 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d37c6c53f2f547d29bc0527e0174dbcc is 50, key is test_row_0/B:col10/1731173212246/Put/seqid=0 2024-11-09T17:26:53,523 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/6123a59e02ca4f70bcd3aa1e6553fc1c is 50, key is test_row_0/A:col10/1731173212246/Put/seqid=0 2024-11-09T17:26:53,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742072_1248 (size=12493) 2024-11-09T17:26:53,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742071_1247 (size=12493) 2024-11-09T17:26:53,610 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:53,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=80 2024-11-09T17:26:53,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:53,610 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:53,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:53,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/26438259a68042258dde5c4c37983a97 is 50, key is test_row_0/A:col10/1731173212260/Put/seqid=0 2024-11-09T17:26:53,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742073_1249 (size=12151) 2024-11-09T17:26:53,932 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d37c6c53f2f547d29bc0527e0174dbcc as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d37c6c53f2f547d29bc0527e0174dbcc 2024-11-09T17:26:53,936 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into d37c6c53f2f547d29bc0527e0174dbcc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:53,936 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:53,936 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173213514; duration=0sec 2024-11-09T17:26:53,936 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:53,936 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:26:53,936 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-09T17:26:53,937 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-09T17:26:53,937 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-09T17:26:53,937 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. because compaction request was cancelled 2024-11-09T17:26:53,937 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:26:53,945 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/6123a59e02ca4f70bcd3aa1e6553fc1c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/6123a59e02ca4f70bcd3aa1e6553fc1c 2024-11-09T17:26:53,950 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 6123a59e02ca4f70bcd3aa1e6553fc1c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:53,950 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:53,950 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173213513; duration=0sec 2024-11-09T17:26:53,950 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:53,950 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:26:54,019 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/26438259a68042258dde5c4c37983a97 2024-11-09T17:26:54,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/fcb2f915d8b44cefb7e40f7616838228 is 50, key is test_row_0/B:col10/1731173212260/Put/seqid=0 2024-11-09T17:26:54,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742074_1250 (size=12151) 2024-11-09T17:26:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:54,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173274396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173274397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173274397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173274397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173274399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,432 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/fcb2f915d8b44cefb7e40f7616838228 2024-11-09T17:26:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/a22f435ecaeb4cbfbb800dfec2b2bc89 is 50, key is test_row_0/C:col10/1731173212260/Put/seqid=0 2024-11-09T17:26:54,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742075_1251 (size=12151) 2024-11-09T17:26:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:54,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173274500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173274500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173274500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173274501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173274704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173274705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173274705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173274705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:54,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/a22f435ecaeb4cbfbb800dfec2b2bc89 2024-11-09T17:26:54,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/26438259a68042258dde5c4c37983a97 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97 2024-11-09T17:26:54,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T17:26:54,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/fcb2f915d8b44cefb7e40f7616838228 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228 2024-11-09T17:26:54,856 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T17:26:54,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/a22f435ecaeb4cbfbb800dfec2b2bc89 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89 2024-11-09T17:26:54,859 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T17:26:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bbbe0e290bd0d6582b636c1e26d5e726 in 1250ms, sequenceid=168, compaction requested=true 2024-11-09T17:26:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-11-09T17:26:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-11-09T17:26:54,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-09T17:26:54,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-11-09T17:26:54,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=79, table=TestAcidGuarantees in 2.5160 sec 2024-11-09T17:26:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:55,009 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:55,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3ce044e079084078865ff9df2d4e2fce is 50, key is test_row_0/A:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742076_1252 (size=12151) 2024-11-09T17:26:55,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3ce044e079084078865ff9df2d4e2fce 2024-11-09T17:26:55,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/0645ad6072d9437d9aa1849ee89c1747 is 50, key is test_row_0/B:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742077_1253 (size=12151) 2024-11-09T17:26:55,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/0645ad6072d9437d9aa1849ee89c1747 2024-11-09T17:26:55,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/fb590ca0779643219b15338e1cc509b0 is 50, key is test_row_0/C:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742078_1254 (size=12151) 2024-11-09T17:26:55,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/fb590ca0779643219b15338e1cc509b0 2024-11-09T17:26:55,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3ce044e079084078865ff9df2d4e2fce as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce 2024-11-09T17:26:55,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T17:26:55,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/0645ad6072d9437d9aa1849ee89c1747 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747 2024-11-09T17:26:55,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T17:26:55,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/fb590ca0779643219b15338e1cc509b0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0 2024-11-09T17:26:55,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T17:26:55,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for bbbe0e290bd0d6582b636c1e26d5e726 in 469ms, sequenceid=184, compaction requested=true 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:55,478 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:55,478 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:55,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:55,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:55,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:26:55,479 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:55,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:26:55,479 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/6123a59e02ca4f70bcd3aa1e6553fc1c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=35.9 K 2024-11-09T17:26:55,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:26:55,479 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:55,479 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d37c6c53f2f547d29bc0527e0174dbcc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=35.9 K 2024-11-09T17:26:55,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6123a59e02ca4f70bcd3aa1e6553fc1c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731173212246 2024-11-09T17:26:55,480 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d37c6c53f2f547d29bc0527e0174dbcc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731173212246 2024-11-09T17:26:55,480 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 26438259a68042258dde5c4c37983a97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731173212260 2024-11-09T17:26:55,480 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3ce044e079084078865ff9df2d4e2fce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:55,480 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting fcb2f915d8b44cefb7e40f7616838228, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731173212260 2024-11-09T17:26:55,480 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0645ad6072d9437d9aa1849ee89c1747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:55,486 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#212 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:55,486 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e938c2367fd445f3a38b12d259f3bb9d is 50, key is test_row_0/A:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,488 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:55,489 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/b4f735037cde4b19a617e9a4a78fc59d is 50, key is test_row_0/B:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742079_1255 (size=12595) 2024-11-09T17:26:55,497 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e938c2367fd445f3a38b12d259f3bb9d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e938c2367fd445f3a38b12d259f3bb9d 2024-11-09T17:26:55,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742080_1256 (size=12595) 2024-11-09T17:26:55,503 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into e938c2367fd445f3a38b12d259f3bb9d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:55,503 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:55,503 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173215478; duration=0sec 2024-11-09T17:26:55,503 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:55,503 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:26:55,503 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:55,504 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48728 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:55,505 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:26:55,505 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:55,505 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cf4b8e1933214feeb9307e33926504c4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=47.6 K 2024-11-09T17:26:55,505 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cf4b8e1933214feeb9307e33926504c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731173211118 2024-11-09T17:26:55,505 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 50fb3b21de7c40088385ad292afe7ed1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731173212246 2024-11-09T17:26:55,505 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting a22f435ecaeb4cbfbb800dfec2b2bc89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731173212260 2024-11-09T17:26:55,506 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting fb590ca0779643219b15338e1cc509b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:55,514 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:55,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/ed8e6b4549d7488a9602ac6d99e10aff is 50, key is test_row_0/C:col10/1731173214397/Put/seqid=0 2024-11-09T17:26:55,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742081_1257 (size=12561) 2024-11-09T17:26:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:55,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:55,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/84f88630e60e49e7a920367d302708f6 is 50, key is test_row_0/A:col10/1731173215638/Put/seqid=0 2024-11-09T17:26:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742082_1258 (size=14541) 2024-11-09T17:26:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,905 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/b4f735037cde4b19a617e9a4a78fc59d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/b4f735037cde4b19a617e9a4a78fc59d 2024-11-09T17:26:55,909 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into b4f735037cde4b19a617e9a4a78fc59d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:55,909 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:55,909 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173215478; duration=0sec 2024-11-09T17:26:55,909 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:55,909 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:26:55,932 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/ed8e6b4549d7488a9602ac6d99e10aff as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/ed8e6b4549d7488a9602ac6d99e10aff 2024-11-09T17:26:55,937 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into ed8e6b4549d7488a9602ac6d99e10aff(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:55,937 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:55,937 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=12, startTime=1731173215478; duration=0sec 2024-11-09T17:26:55,937 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:55,937 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:26:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173275954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173275955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173275956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:55,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173275957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/84f88630e60e49e7a920367d302708f6 2024-11-09T17:26:56,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d84cdb953ea84ac18ebdd478fc411203 is 50, key is test_row_0/B:col10/1731173215638/Put/seqid=0 2024-11-09T17:26:56,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742083_1259 (size=12151) 2024-11-09T17:26:56,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173276257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173276257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173276259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173276260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173276407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,409 DEBUG [Thread-1017 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4141 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:26:56,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-09T17:26:56,451 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 79 completed 2024-11-09T17:26:56,452 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-11-09T17:26:56,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:56,453 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:56,453 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:56,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:56,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d84cdb953ea84ac18ebdd478fc411203 2024-11-09T17:26:56,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/04374a7d959c40ffa0257e04277fac8a is 50, key is test_row_0/C:col10/1731173215638/Put/seqid=0 2024-11-09T17:26:56,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742084_1260 (size=12151) 2024-11-09T17:26:56,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:56,605 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-09T17:26:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:56,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:56,757 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-09T17:26:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173276759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173276762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173276762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173276765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/04374a7d959c40ffa0257e04277fac8a 2024-11-09T17:26:56,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/84f88630e60e49e7a920367d302708f6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6 2024-11-09T17:26:56,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6, entries=200, sequenceid=209, filesize=14.2 K 2024-11-09T17:26:56,910 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:56,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/d84cdb953ea84ac18ebdd478fc411203 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203 2024-11-09T17:26:56,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-09T17:26:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:56,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:56,910 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:56,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203, entries=150, sequenceid=209, filesize=11.9 K 2024-11-09T17:26:56,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/04374a7d959c40ffa0257e04277fac8a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a 2024-11-09T17:26:56,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-09T17:26:56,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for bbbe0e290bd0d6582b636c1e26d5e726 in 1281ms, sequenceid=209, compaction requested=false 2024-11-09T17:26:56,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:57,062 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-09T17:26:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:57,063 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:26:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:57,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/cf417b4aef48420ba83511e3b9881360 is 50, key is test_row_0/A:col10/1731173215645/Put/seqid=0 2024-11-09T17:26:57,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742085_1261 (size=12151) 2024-11-09T17:26:57,472 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/cf417b4aef48420ba83511e3b9881360 2024-11-09T17:26:57,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1fbdfd66d7c44eff8ea896a4e9d03941 is 50, key is test_row_0/B:col10/1731173215645/Put/seqid=0 2024-11-09T17:26:57,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742086_1262 (size=12151) 2024-11-09T17:26:57,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:57,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173277792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173277792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173277793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173277793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,895 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1fbdfd66d7c44eff8ea896a4e9d03941 2024-11-09T17:26:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173277896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173277896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173277897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:57,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173277898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:57,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cb27fb6487824910825e0feaf1ef7dbd is 50, key is test_row_0/C:col10/1731173215645/Put/seqid=0 2024-11-09T17:26:57,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742087_1263 (size=12151) 2024-11-09T17:26:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173278099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173278100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173278100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173278104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,306 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cb27fb6487824910825e0feaf1ef7dbd 2024-11-09T17:26:58,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/cf417b4aef48420ba83511e3b9881360 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360 2024-11-09T17:26:58,314 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T17:26:58,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1fbdfd66d7c44eff8ea896a4e9d03941 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941 2024-11-09T17:26:58,318 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T17:26:58,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/cb27fb6487824910825e0feaf1ef7dbd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd 2024-11-09T17:26:58,322 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T17:26:58,323 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bbbe0e290bd0d6582b636c1e26d5e726 in 1259ms, sequenceid=223, compaction requested=true 2024-11-09T17:26:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:58,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-09T17:26:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-11-09T17:26:58,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-09T17:26:58,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8700 sec 2024-11-09T17:26:58,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 1.8730 sec 2024-11-09T17:26:58,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:58,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:58,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:58,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/317208807000493cb8b967362c6b6131 is 50, key is test_row_0/A:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173278411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173278411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742088_1264 (size=12151) 2024-11-09T17:26:58,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/317208807000493cb8b967362c6b6131 2024-11-09T17:26:58,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173278412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173278414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/6540e9db266043718f014caf34549e17 is 50, key is test_row_0/B:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:58,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742089_1265 (size=12151) 2024-11-09T17:26:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173278515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173278515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173278516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173278519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-09T17:26:58,556 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 81 completed 2024-11-09T17:26:58,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:26:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-09T17:26:58,558 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:26:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:26:58,559 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:26:58,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:26:58,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:26:58,710 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-09T17:26:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:58,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173278717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173278717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173278719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173278720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:26:58,862 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:58,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-09T17:26:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:58,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:58,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/6540e9db266043718f014caf34549e17 2024-11-09T17:26:58,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3c06a634c3914a41b0b213092eb1711f is 50, key is test_row_0/C:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:58,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742090_1266 (size=12151) 2024-11-09T17:26:59,015 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-09T17:26:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:59,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173279020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173279022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173279022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173279026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:26:59,167 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-09T17:26:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:26:59,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3c06a634c3914a41b0b213092eb1711f 2024-11-09T17:26:59,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/317208807000493cb8b967362c6b6131 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131 2024-11-09T17:26:59,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131, entries=150, sequenceid=249, filesize=11.9 K 2024-11-09T17:26:59,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/6540e9db266043718f014caf34549e17 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17 2024-11-09T17:26:59,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17, entries=150, sequenceid=249, filesize=11.9 K 2024-11-09T17:26:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/3c06a634c3914a41b0b213092eb1711f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f 2024-11-09T17:26:59,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f, entries=150, sequenceid=249, filesize=11.9 K 2024-11-09T17:26:59,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bbbe0e290bd0d6582b636c1e26d5e726 in 897ms, sequenceid=249, compaction requested=true 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:59,302 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:26:59,302 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:26:59,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:59,303 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51438 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:59,303 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:59,303 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:26:59,303 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:26:59,304 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,304 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,304 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/b4f735037cde4b19a617e9a4a78fc59d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=47.9 K 2024-11-09T17:26:59,304 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e938c2367fd445f3a38b12d259f3bb9d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=50.2 K 2024-11-09T17:26:59,304 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e938c2367fd445f3a38b12d259f3bb9d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:59,304 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b4f735037cde4b19a617e9a4a78fc59d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:59,304 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d84cdb953ea84ac18ebdd478fc411203, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731173215021 2024-11-09T17:26:59,305 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 84f88630e60e49e7a920367d302708f6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731173215021 2024-11-09T17:26:59,305 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1fbdfd66d7c44eff8ea896a4e9d03941, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731173215645 2024-11-09T17:26:59,305 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting cf417b4aef48420ba83511e3b9881360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731173215645 2024-11-09T17:26:59,305 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6540e9db266043718f014caf34549e17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:26:59,305 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 317208807000493cb8b967362c6b6131, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:26:59,317 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:59,317 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:59,317 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/4894c66b8ddf4a7f991541596f00d785 is 50, key is test_row_0/A:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:59,317 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/efc2eb94469a4d4d8c7a05b1dd3d64f5 is 50, key is test_row_0/B:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:59,320 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-09T17:26:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,320 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T17:26:59,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:26:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:26:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:26:59,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:26:59,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742091_1267 (size=12731) 2024-11-09T17:26:59,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742092_1268 (size=12731) 2024-11-09T17:26:59,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7f4fba5db9d64213af0ad78a49b504c5 is 50, key is test_row_0/A:col10/1731173218413/Put/seqid=0 2024-11-09T17:26:59,345 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/efc2eb94469a4d4d8c7a05b1dd3d64f5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/efc2eb94469a4d4d8c7a05b1dd3d64f5 2024-11-09T17:26:59,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742093_1269 (size=9857) 2024-11-09T17:26:59,349 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into efc2eb94469a4d4d8c7a05b1dd3d64f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:59,349 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:59,349 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=12, startTime=1731173219302; duration=0sec 2024-11-09T17:26:59,349 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:26:59,349 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:26:59,349 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:26:59,352 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:26:59,352 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:26:59,352 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:26:59,352 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/ed8e6b4549d7488a9602ac6d99e10aff, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=47.9 K 2024-11-09T17:26:59,352 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting ed8e6b4549d7488a9602ac6d99e10aff, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731173214395 2024-11-09T17:26:59,353 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 04374a7d959c40ffa0257e04277fac8a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731173215021 2024-11-09T17:26:59,353 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cb27fb6487824910825e0feaf1ef7dbd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731173215645 2024-11-09T17:26:59,353 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3c06a634c3914a41b0b213092eb1711f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:26:59,372 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:26:59,372 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4f84b163494c4d949fba242bb37df5e2 is 50, key is test_row_0/C:col10/1731173218406/Put/seqid=0 2024-11-09T17:26:59,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742094_1270 (size=12697) 2024-11-09T17:26:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:26:59,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:26:59,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173279545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173279546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173279547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173279548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173279649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173279649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173279650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173279650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:26:59,736 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/4894c66b8ddf4a7f991541596f00d785 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4894c66b8ddf4a7f991541596f00d785 2024-11-09T17:26:59,741 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 4894c66b8ddf4a7f991541596f00d785(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:59,741 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:59,741 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=12, startTime=1731173219302; duration=0sec 2024-11-09T17:26:59,741 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:59,741 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:26:59,746 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7f4fba5db9d64213af0ad78a49b504c5 2024-11-09T17:26:59,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f21e82c1d03411c949855640ec130a7 is 50, key is test_row_0/B:col10/1731173218413/Put/seqid=0 2024-11-09T17:26:59,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742095_1271 (size=9857) 2024-11-09T17:26:59,780 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4f84b163494c4d949fba242bb37df5e2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4f84b163494c4d949fba242bb37df5e2 2024-11-09T17:26:59,785 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into 4f84b163494c4d949fba242bb37df5e2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:26:59,785 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:26:59,785 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=12, startTime=1731173219302; duration=0sec 2024-11-09T17:26:59,785 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:26:59,785 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:26:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173279852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173279852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173279852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:26:59,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:26:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173279852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173280154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173280154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173280155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,158 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f21e82c1d03411c949855640ec130a7 2024-11-09T17:27:00,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173280156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4c53926fb3534795991e35f3b8a1c836 is 50, key is test_row_0/C:col10/1731173218413/Put/seqid=0 2024-11-09T17:27:00,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742096_1272 (size=9857) 2024-11-09T17:27:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45214 deadline: 1731173280410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,411 DEBUG [Thread-1017 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8143 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:00,567 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4c53926fb3534795991e35f3b8a1c836 2024-11-09T17:27:00,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/7f4fba5db9d64213af0ad78a49b504c5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5 2024-11-09T17:27:00,574 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5, entries=100, sequenceid=259, filesize=9.6 K 2024-11-09T17:27:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f21e82c1d03411c949855640ec130a7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7 2024-11-09T17:27:00,579 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7, entries=100, sequenceid=259, filesize=9.6 K 2024-11-09T17:27:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4c53926fb3534795991e35f3b8a1c836 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836 2024-11-09T17:27:00,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836, entries=100, sequenceid=259, filesize=9.6 K 2024-11-09T17:27:00,584 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for bbbe0e290bd0d6582b636c1e26d5e726 in 1264ms, sequenceid=259, compaction requested=false 2024-11-09T17:27:00,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:00,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:00,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-09T17:27:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-11-09T17:27:00,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-11-09T17:27:00,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0270 sec 2024-11-09T17:27:00,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 2.0310 sec 2024-11-09T17:27:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:00,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:00,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-09T17:27:00,662 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 83 completed 2024-11-09T17:27:00,663 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:00,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/fb2497174e2244ba8c3d916b5937952d is 50, key is test_row_0/A:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-09T17:27:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:00,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:00,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:00,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173280663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173280664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742097_1273 (size=14741) 2024-11-09T17:27:00,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/fb2497174e2244ba8c3d916b5937952d 2024-11-09T17:27:00,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173280666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173280667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f159951db574144962ef8c84b2c219f is 50, key is test_row_0/B:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:00,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742098_1274 (size=12301) 2024-11-09T17:27:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:00,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173280768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173280768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173280770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173280770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,816 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:00,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:00,968 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:00,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:00,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:00,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:00,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:00,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173280970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173280971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173280972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:00,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:00,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173280972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f159951db574144962ef8c84b2c219f 2024-11-09T17:27:01,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4e81ebd16baf44db83d9aab8586ff80f is 50, key is test_row_0/C:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:01,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742099_1275 (size=12301) 2024-11-09T17:27:01,120 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:01,273 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:01,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173281273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173281273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173281275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173281276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,425 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:01,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4e81ebd16baf44db83d9aab8586ff80f 2024-11-09T17:27:01,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/fb2497174e2244ba8c3d916b5937952d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d 2024-11-09T17:27:01,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d, entries=200, sequenceid=289, filesize=14.4 K 2024-11-09T17:27:01,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/1f159951db574144962ef8c84b2c219f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f 2024-11-09T17:27:01,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-09T17:27:01,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/4e81ebd16baf44db83d9aab8586ff80f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f 2024-11-09T17:27:01,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f, entries=150, sequenceid=289, filesize=12.0 K 2024-11-09T17:27:01,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for bbbe0e290bd0d6582b636c1e26d5e726 in 847ms, sequenceid=289, compaction requested=true 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:01,506 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:01,506 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:01,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:01,507 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:01,507 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:27:01,507 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,507 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4894c66b8ddf4a7f991541596f00d785, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=36.5 K 2024-11-09T17:27:01,508 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4894c66b8ddf4a7f991541596f00d785, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:27:01,508 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7f4fba5db9d64213af0ad78a49b504c5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731173218411 2024-11-09T17:27:01,508 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting fb2497174e2244ba8c3d916b5937952d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:01,509 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:01,509 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:27:01,509 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,509 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/efc2eb94469a4d4d8c7a05b1dd3d64f5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=34.1 K 2024-11-09T17:27:01,510 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting efc2eb94469a4d4d8c7a05b1dd3d64f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:27:01,510 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1f21e82c1d03411c949855640ec130a7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731173218411 2024-11-09T17:27:01,510 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1f159951db574144962ef8c84b2c219f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:01,522 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:01,523 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:01,523 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/4ba6d7e185374e9980f0e071683e9246 is 50, key is test_row_0/A:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:01,523 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/9765b8fd426b442aae5a4650f4b67948 is 50, key is test_row_0/B:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:01,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742100_1276 (size=12983) 2024-11-09T17:27:01,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742101_1277 (size=12983) 2024-11-09T17:27:01,541 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/9765b8fd426b442aae5a4650f4b67948 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9765b8fd426b442aae5a4650f4b67948 2024-11-09T17:27:01,545 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into 9765b8fd426b442aae5a4650f4b67948(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:01,545 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:01,545 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173221506; duration=0sec 2024-11-09T17:27:01,545 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:01,545 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:27:01,545 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:01,546 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:01,546 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:27:01,546 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,546 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4f84b163494c4d949fba242bb37df5e2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=34.0 K 2024-11-09T17:27:01,547 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4f84b163494c4d949fba242bb37df5e2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731173217792 2024-11-09T17:27:01,547 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4c53926fb3534795991e35f3b8a1c836, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731173218411 2024-11-09T17:27:01,547 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4e81ebd16baf44db83d9aab8586ff80f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:01,553 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#235 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:01,554 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/7f3b6ec830c946938794f14640d185e8 is 50, key is test_row_0/C:col10/1731173219544/Put/seqid=0 2024-11-09T17:27:01,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742102_1278 (size=12949) 2024-11-09T17:27:01,561 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/7f3b6ec830c946938794f14640d185e8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/7f3b6ec830c946938794f14640d185e8 2024-11-09T17:27:01,566 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into 7f3b6ec830c946938794f14640d185e8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:01,566 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:01,566 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=13, startTime=1731173221506; duration=0sec 2024-11-09T17:27:01,566 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:01,566 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:27:01,577 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:01,578 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:01,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/8dd16879646240fca09819c738cc12c8 is 50, key is test_row_0/A:col10/1731173220664/Put/seqid=0 2024-11-09T17:27:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742103_1279 (size=12301) 2024-11-09T17:27:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:01,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:01,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:01,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173281801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173281804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173281805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173281805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173281906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173281908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173281912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173281912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:01,942 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/4ba6d7e185374e9980f0e071683e9246 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4ba6d7e185374e9980f0e071683e9246 2024-11-09T17:27:01,947 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 4ba6d7e185374e9980f0e071683e9246(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:01,947 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:01,947 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173221506; duration=0sec 2024-11-09T17:27:01,947 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:01,947 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:27:01,986 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/8dd16879646240fca09819c738cc12c8 2024-11-09T17:27:01,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/26fb9cbad15c4896adf0287a144094d7 is 50, key is test_row_0/B:col10/1731173220664/Put/seqid=0 2024-11-09T17:27:02,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742104_1280 (size=12301) 2024-11-09T17:27:02,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173282112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173282113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173282113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173282114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,401 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/26fb9cbad15c4896adf0287a144094d7 2024-11-09T17:27:02,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/6338aa7004f34e9ebb8fbb8acc30363b is 50, key is test_row_0/C:col10/1731173220664/Put/seqid=0 2024-11-09T17:27:02,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173282414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173282416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173282417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173282417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742105_1281 (size=12301) 2024-11-09T17:27:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:02,828 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/6338aa7004f34e9ebb8fbb8acc30363b 2024-11-09T17:27:02,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/8dd16879646240fca09819c738cc12c8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8 2024-11-09T17:27:02,837 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8, entries=150, sequenceid=300, filesize=12.0 K 2024-11-09T17:27:02,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/26fb9cbad15c4896adf0287a144094d7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7 2024-11-09T17:27:02,842 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7, entries=150, sequenceid=300, filesize=12.0 K 2024-11-09T17:27:02,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/6338aa7004f34e9ebb8fbb8acc30363b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b 2024-11-09T17:27:02,846 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b, entries=150, sequenceid=300, filesize=12.0 K 2024-11-09T17:27:02,847 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3040): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for bbbe0e290bd0d6582b636c1e26d5e726 in 1269ms, sequenceid=300, compaction requested=false 2024-11-09T17:27:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:02,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-09T17:27:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-11-09T17:27:02,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-09T17:27:02,849 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1840 sec 2024-11-09T17:27:02,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.1860 sec 2024-11-09T17:27:02,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:02,920 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:02,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:02,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173282922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173282923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173282923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173282924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:02,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e94a5853702447f8935c3a256011f40e is 50, key is test_row_0/A:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:02,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742106_1282 (size=14741) 2024-11-09T17:27:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173283026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173283026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173283026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173283228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173283229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173283229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e94a5853702447f8935c3a256011f40e 2024-11-09T17:27:03,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/38686951d8db416685a1fd886be4616d is 50, key is test_row_0/B:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:03,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742107_1283 (size=12301) 2024-11-09T17:27:03,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173283531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173283531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173283531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:03,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/38686951d8db416685a1fd886be4616d 2024-11-09T17:27:03,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d3d6bd8ed6d541b987b1114ef88d2af8 is 50, key is test_row_0/C:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:03,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742108_1284 (size=12301) 2024-11-09T17:27:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173283925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:04,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173284034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173284038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173284038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:04,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d3d6bd8ed6d541b987b1114ef88d2af8 2024-11-09T17:27:04,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/e94a5853702447f8935c3a256011f40e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e 2024-11-09T17:27:04,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e, entries=200, sequenceid=331, filesize=14.4 K 2024-11-09T17:27:04,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/38686951d8db416685a1fd886be4616d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d 2024-11-09T17:27:04,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T17:27:04,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/d3d6bd8ed6d541b987b1114ef88d2af8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8 2024-11-09T17:27:04,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8, entries=150, sequenceid=331, filesize=12.0 K 2024-11-09T17:27:04,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for bbbe0e290bd0d6582b636c1e26d5e726 in 1271ms, sequenceid=331, compaction requested=true 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:04,191 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:04,191 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:04,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:04,192 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:04,193 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:27:04,193 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:04,193 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4ba6d7e185374e9980f0e071683e9246, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=39.1 K 2024-11-09T17:27:04,193 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:04,193 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:27:04,193 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:04,193 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9765b8fd426b442aae5a4650f4b67948, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=36.7 K 2024-11-09T17:27:04,193 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4ba6d7e185374e9980f0e071683e9246, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:04,193 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9765b8fd426b442aae5a4650f4b67948, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:04,194 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8dd16879646240fca09819c738cc12c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731173220663 2024-11-09T17:27:04,194 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 26fb9cbad15c4896adf0287a144094d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731173220663 2024-11-09T17:27:04,194 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e94a5853702447f8935c3a256011f40e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:04,194 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 38686951d8db416685a1fd886be4616d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:04,200 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:04,200 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/83f5067fdec14ba9afc079ca0ad0457f is 50, key is test_row_0/A:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:04,228 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:04,229 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/50e1d40babed4510a48f361e477c19f4 is 50, key is test_row_0/B:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742109_1285 (size=13085) 2024-11-09T17:27:04,237 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/83f5067fdec14ba9afc079ca0ad0457f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/83f5067fdec14ba9afc079ca0ad0457f 2024-11-09T17:27:04,241 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 83f5067fdec14ba9afc079ca0ad0457f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:04,242 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:04,242 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173224191; duration=0sec 2024-11-09T17:27:04,242 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:04,242 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:27:04,242 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:04,243 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:04,243 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:27:04,243 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:04,243 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/7f3b6ec830c946938794f14640d185e8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=36.7 K 2024-11-09T17:27:04,243 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7f3b6ec830c946938794f14640d185e8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731173219544 2024-11-09T17:27:04,243 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6338aa7004f34e9ebb8fbb8acc30363b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731173220663 2024-11-09T17:27:04,244 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d3d6bd8ed6d541b987b1114ef88d2af8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:04,252 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:04,252 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/2156ab823ed64efe924e24203e294964 is 50, key is test_row_0/C:col10/1731173221804/Put/seqid=0 2024-11-09T17:27:04,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742110_1286 (size=13085) 2024-11-09T17:27:04,265 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/50e1d40babed4510a48f361e477c19f4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/50e1d40babed4510a48f361e477c19f4 2024-11-09T17:27:04,270 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into 50e1d40babed4510a48f361e477c19f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:04,270 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:04,270 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173224191; duration=0sec 2024-11-09T17:27:04,270 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:04,270 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:27:04,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742111_1287 (size=13051) 2024-11-09T17:27:04,282 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/2156ab823ed64efe924e24203e294964 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2156ab823ed64efe924e24203e294964 2024-11-09T17:27:04,289 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into 2156ab823ed64efe924e24203e294964(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:04,289 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:04,290 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=13, startTime=1731173224191; duration=0sec 2024-11-09T17:27:04,290 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:04,290 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:27:04,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-09T17:27:04,768 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 85 completed 2024-11-09T17:27:04,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-09T17:27:04,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:04,770 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:04,771 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:04,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:04,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:04,922 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:04,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-09T17:27:04,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:04,922 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:04,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:04,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3f50f514c40f43858df6d3a532d47e4c is 50, key is test_row_0/A:col10/1731173222922/Put/seqid=0 2024-11-09T17:27:04,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742112_1288 (size=9857) 2024-11-09T17:27:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:05,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. as already flushing 2024-11-09T17:27:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173285066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173285066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173285067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173285170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173285170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173285170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,331 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3f50f514c40f43858df6d3a532d47e4c 2024-11-09T17:27:05,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/9f20867898e946be8a623f6de84d0d1c is 50, key is test_row_0/B:col10/1731173222922/Put/seqid=0 2024-11-09T17:27:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742113_1289 (size=9857) 2024-11-09T17:27:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:05,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173285372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173285373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173285373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45230 deadline: 1731173285675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45216 deadline: 1731173285676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45210 deadline: 1731173285676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,740 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/9f20867898e946be8a623f6de84d0d1c 2024-11-09T17:27:05,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/67076fee7e1f414598518a02e7e76ba6 is 50, key is test_row_0/C:col10/1731173222922/Put/seqid=0 2024-11-09T17:27:05,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742114_1290 (size=9857) 2024-11-09T17:27:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:05,918 DEBUG [Thread-1028 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0eb04aeb to 127.0.0.1:57222 2024-11-09T17:27:05,918 DEBUG [Thread-1036 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1730a60f to 127.0.0.1:57222 2024-11-09T17:27:05,918 DEBUG [Thread-1036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:05,918 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:05,919 DEBUG [Thread-1032 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d68f787 to 127.0.0.1:57222 2024-11-09T17:27:05,919 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:05,920 DEBUG [Thread-1030 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6a0e9c8f to 127.0.0.1:57222 2024-11-09T17:27:05,920 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:05,920 DEBUG [Thread-1034 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10e6bf6a to 127.0.0.1:57222 2024-11-09T17:27:05,920 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:05,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45182 deadline: 1731173285934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:05,935 DEBUG [Thread-1025 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4130 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:06,151 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/67076fee7e1f414598518a02e7e76ba6 2024-11-09T17:27:06,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/3f50f514c40f43858df6d3a532d47e4c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c 2024-11-09T17:27:06,161 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c, entries=100, sequenceid=340, filesize=9.6 K 2024-11-09T17:27:06,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/9f20867898e946be8a623f6de84d0d1c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c 2024-11-09T17:27:06,165 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c, entries=100, sequenceid=340, filesize=9.6 K 2024-11-09T17:27:06,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/67076fee7e1f414598518a02e7e76ba6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6 2024-11-09T17:27:06,169 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6, entries=100, sequenceid=340, filesize=9.6 K 2024-11-09T17:27:06,170 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3040): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for bbbe0e290bd0d6582b636c1e26d5e726 in 1248ms, sequenceid=340, compaction requested=false 2024-11-09T17:27:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:06,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-09T17:27:06,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-11-09T17:27:06,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T17:27:06,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-09T17:27:06,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.4030 sec 2024-11-09T17:27:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:06,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=201.27 KB heapSize=528.09 KB 2024-11-09T17:27:06,180 DEBUG [Thread-1023 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x43f04e0e to 127.0.0.1:57222 2024-11-09T17:27:06,180 DEBUG [Thread-1021 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x08d0caa5 to 127.0.0.1:57222 2024-11-09T17:27:06,180 DEBUG [Thread-1023 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:06,180 DEBUG [Thread-1021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:06,180 DEBUG [Thread-1019 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0d296fed to 127.0.0.1:57222 2024-11-09T17:27:06,180 DEBUG [Thread-1019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:06,180 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:06,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:06,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/00ae9e39e2724dfeb2b4e326d26db863 is 50, key is test_row_0/A:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:06,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742115_1291 (size=12301) 2024-11-09T17:27:06,581 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:27:06,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/00ae9e39e2724dfeb2b4e326d26db863 2024-11-09T17:27:06,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/3c4c32432438424ea47348f160d4f98c is 50, key is test_row_0/B:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:06,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742116_1292 (size=12301) 2024-11-09T17:27:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-09T17:27:06,875 INFO [Thread-1027 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 87 completed 2024-11-09T17:27:07,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/3c4c32432438424ea47348f160d4f98c 2024-11-09T17:27:07,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/404d2ec32c8f4c7f8e69e77637bd53f2 is 50, key is test_row_0/C:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742117_1293 (size=12301) 2024-11-09T17:27:07,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/404d2ec32c8f4c7f8e69e77637bd53f2 2024-11-09T17:27:07,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/00ae9e39e2724dfeb2b4e326d26db863 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863 2024-11-09T17:27:07,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T17:27:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/3c4c32432438424ea47348f160d4f98c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c 2024-11-09T17:27:07,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T17:27:07,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/404d2ec32c8f4c7f8e69e77637bd53f2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2 2024-11-09T17:27:07,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T17:27:07,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~201.27 KB/206100, heapSize ~528.05 KB/540720, currentSize=0 B/0 for bbbe0e290bd0d6582b636c1e26d5e726 in 1274ms, sequenceid=373, compaction requested=true 2024-11-09T17:27:07,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:07,455 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:07,455 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bbbe0e290bd0d6582b636c1e26d5e726:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:07,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:07,456 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:07,456 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/A is initiating minor compaction (all files) 2024-11-09T17:27:07,456 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:07,456 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/A in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:07,456 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/B is initiating minor compaction (all files) 2024-11-09T17:27:07,456 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/B in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:07,456 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/83f5067fdec14ba9afc079ca0ad0457f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=34.4 K 2024-11-09T17:27:07,456 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/50e1d40babed4510a48f361e477c19f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=34.4 K 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 83f5067fdec14ba9afc079ca0ad0457f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 50e1d40babed4510a48f361e477c19f4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9f20867898e946be8a623f6de84d0d1c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731173222922 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3f50f514c40f43858df6d3a532d47e4c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731173222922 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 3c4c32432438424ea47348f160d4f98c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731173225062 2024-11-09T17:27:07,457 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 00ae9e39e2724dfeb2b4e326d26db863, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731173225062 2024-11-09T17:27:07,465 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#A#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:07,465 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#B#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:07,466 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/84ab8583b09a4c9eab460a24695fbd5c is 50, key is test_row_0/A:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:07,466 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/544245b1e993475cbfb783b86fe1bd89 is 50, key is test_row_0/B:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742119_1295 (size=13187) 2024-11-09T17:27:07,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742118_1294 (size=13187) 2024-11-09T17:27:07,881 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/544245b1e993475cbfb783b86fe1bd89 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/544245b1e993475cbfb783b86fe1bd89 2024-11-09T17:27:07,881 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/84ab8583b09a4c9eab460a24695fbd5c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84ab8583b09a4c9eab460a24695fbd5c 2024-11-09T17:27:07,886 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/A of bbbe0e290bd0d6582b636c1e26d5e726 into 84ab8583b09a4c9eab460a24695fbd5c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:07,886 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/B of bbbe0e290bd0d6582b636c1e26d5e726 into 544245b1e993475cbfb783b86fe1bd89(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:07,886 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:07,886 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:07,886 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/B, priority=13, startTime=1731173227455; duration=0sec 2024-11-09T17:27:07,886 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/A, priority=13, startTime=1731173227455; duration=0sec 2024-11-09T17:27:07,887 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:07,887 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:B 2024-11-09T17:27:07,887 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:07,887 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:A 2024-11-09T17:27:07,887 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:07,888 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:07,888 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): bbbe0e290bd0d6582b636c1e26d5e726/C is initiating minor compaction (all files) 2024-11-09T17:27:07,888 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of bbbe0e290bd0d6582b636c1e26d5e726/C in TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:07,888 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2156ab823ed64efe924e24203e294964, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp, totalSize=34.4 K 2024-11-09T17:27:07,888 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2156ab823ed64efe924e24203e294964, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731173221804 2024-11-09T17:27:07,889 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 67076fee7e1f414598518a02e7e76ba6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731173222922 2024-11-09T17:27:07,889 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 404d2ec32c8f4c7f8e69e77637bd53f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731173225062 2024-11-09T17:27:07,897 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bbbe0e290bd0d6582b636c1e26d5e726#C#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:07,897 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/44e269f1eda14fc6b3fa5a2f9995c84a is 50, key is test_row_0/C:col10/1731173226179/Put/seqid=0 2024-11-09T17:27:07,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742120_1296 (size=13153) 2024-11-09T17:27:08,313 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/44e269f1eda14fc6b3fa5a2f9995c84a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/44e269f1eda14fc6b3fa5a2f9995c84a 2024-11-09T17:27:08,319 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in bbbe0e290bd0d6582b636c1e26d5e726/C of bbbe0e290bd0d6582b636c1e26d5e726 into 44e269f1eda14fc6b3fa5a2f9995c84a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:08,319 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:08,319 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726., storeName=bbbe0e290bd0d6582b636c1e26d5e726/C, priority=13, startTime=1731173227455; duration=0sec 2024-11-09T17:27:08,319 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:08,319 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bbbe0e290bd0d6582b636c1e26d5e726:C 2024-11-09T17:27:09,973 DEBUG [Thread-1025 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x560ec309 to 127.0.0.1:57222 2024-11-09T17:27:09,973 DEBUG [Thread-1025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:10,428 DEBUG [Thread-1017 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6862e3ce to 127.0.0.1:57222 2024-11-09T17:27:10,428 DEBUG [Thread-1017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6494 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 7116 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6520 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6460 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 7121 2024-11-09T17:27:10,429 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:27:10,430 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:27:10,430 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3a569490 to 127.0.0.1:57222 2024-11-09T17:27:10,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:10,431 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:27:10,431 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:27:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:10,436 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173230435"}]},"ts":"1731173230435"} 2024-11-09T17:27:10,436 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:27:10,487 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:27:10,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:27:10,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, UNASSIGN}] 2024-11-09T17:27:10,489 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=91, ppid=90, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, UNASSIGN 2024-11-09T17:27:10,490 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=bbbe0e290bd0d6582b636c1e26d5e726, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:10,491 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:27:10,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; CloseRegionProcedure bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:10,642 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:10,643 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(124): Close bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:10,643 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:27:10,644 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1681): Closing bbbe0e290bd0d6582b636c1e26d5e726, disabling compactions & flushes 2024-11-09T17:27:10,644 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:10,644 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:10,644 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. after waiting 0 ms 2024-11-09T17:27:10,644 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:10,644 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(2837): Flushing bbbe0e290bd0d6582b636c1e26d5e726 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-09T17:27:10,645 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=A 2024-11-09T17:27:10,645 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:10,645 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=B 2024-11-09T17:27:10,646 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:10,646 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bbbe0e290bd0d6582b636c1e26d5e726, store=C 2024-11-09T17:27:10,646 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:10,653 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/c39081c5bbcd44648eef59d41636245a is 50, key is test_row_1/A:col10/1731173230425/Put/seqid=0 2024-11-09T17:27:10,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742121_1297 (size=7415) 2024-11-09T17:27:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:11,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:11,059 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/c39081c5bbcd44648eef59d41636245a 2024-11-09T17:27:11,072 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/337c8803373b4610b6f0be6593525dd1 is 50, key is test_row_1/B:col10/1731173230425/Put/seqid=0 2024-11-09T17:27:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742122_1298 (size=7415) 2024-11-09T17:27:11,478 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/337c8803373b4610b6f0be6593525dd1 2024-11-09T17:27:11,489 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/5da55c9d7cb741f0a3fec9e21291f336 is 50, key is test_row_1/C:col10/1731173230425/Put/seqid=0 2024-11-09T17:27:11,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742123_1299 (size=7415) 2024-11-09T17:27:11,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:11,853 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:11,895 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/5da55c9d7cb741f0a3fec9e21291f336 2024-11-09T17:27:11,904 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/A/c39081c5bbcd44648eef59d41636245a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/c39081c5bbcd44648eef59d41636245a 2024-11-09T17:27:11,909 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/c39081c5bbcd44648eef59d41636245a, entries=50, sequenceid=381, filesize=7.2 K 2024-11-09T17:27:11,910 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/B/337c8803373b4610b6f0be6593525dd1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/337c8803373b4610b6f0be6593525dd1 2024-11-09T17:27:11,914 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/337c8803373b4610b6f0be6593525dd1, entries=50, sequenceid=381, filesize=7.2 K 2024-11-09T17:27:11,915 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/.tmp/C/5da55c9d7cb741f0a3fec9e21291f336 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/5da55c9d7cb741f0a3fec9e21291f336 2024-11-09T17:27:11,919 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/5da55c9d7cb741f0a3fec9e21291f336, entries=50, sequenceid=381, filesize=7.2 K 2024-11-09T17:27:11,920 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(3040): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for bbbe0e290bd0d6582b636c1e26d5e726 in 1276ms, sequenceid=381, compaction requested=false 2024-11-09T17:27:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/088091985d7549688b01f2ea3cfe4d74, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70540dea243e49c19dc97f2fcacb5d5c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/6123a59e02ca4f70bcd3aa1e6553fc1c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e938c2367fd445f3a38b12d259f3bb9d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4894c66b8ddf4a7f991541596f00d785, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4ba6d7e185374e9980f0e071683e9246, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/83f5067fdec14ba9afc079ca0ad0457f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863] to archive 2024-11-09T17:27:11,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:11,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70b178e811da468eb2bc669017dc3dbb 2024-11-09T17:27:11,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7cfe2979e6344a99a005082b9ecde6d6 2024-11-09T17:27:11,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/32b580b169e242edaef7aec352faf91d 2024-11-09T17:27:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/01fae30aeb4a4648a180e0645566d716 2024-11-09T17:27:11,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/088091985d7549688b01f2ea3cfe4d74 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/088091985d7549688b01f2ea3cfe4d74 2024-11-09T17:27:11,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/526834907d3844d4b6bf2ebf3613de9d 2024-11-09T17:27:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/2338bdbb82e246c0a66f4f97791b90cf 2024-11-09T17:27:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70540dea243e49c19dc97f2fcacb5d5c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/70540dea243e49c19dc97f2fcacb5d5c 2024-11-09T17:27:11,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e0a55c5df6a04015ad1b2e8050f5d4c9 2024-11-09T17:27:11,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7503d2c583d943af84cdb226d25e8b6c 2024-11-09T17:27:11,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/6123a59e02ca4f70bcd3aa1e6553fc1c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/6123a59e02ca4f70bcd3aa1e6553fc1c 2024-11-09T17:27:11,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/26438259a68042258dde5c4c37983a97 2024-11-09T17:27:11,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e938c2367fd445f3a38b12d259f3bb9d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e938c2367fd445f3a38b12d259f3bb9d 2024-11-09T17:27:11,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3ce044e079084078865ff9df2d4e2fce 2024-11-09T17:27:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84f88630e60e49e7a920367d302708f6 2024-11-09T17:27:11,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/cf417b4aef48420ba83511e3b9881360 2024-11-09T17:27:11,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4894c66b8ddf4a7f991541596f00d785 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4894c66b8ddf4a7f991541596f00d785 2024-11-09T17:27:11,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/317208807000493cb8b967362c6b6131 2024-11-09T17:27:11,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/7f4fba5db9d64213af0ad78a49b504c5 2024-11-09T17:27:11,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/fb2497174e2244ba8c3d916b5937952d 2024-11-09T17:27:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4ba6d7e185374e9980f0e071683e9246 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/4ba6d7e185374e9980f0e071683e9246 2024-11-09T17:27:11,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/8dd16879646240fca09819c738cc12c8 2024-11-09T17:27:11,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/e94a5853702447f8935c3a256011f40e 2024-11-09T17:27:11,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/83f5067fdec14ba9afc079ca0ad0457f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/83f5067fdec14ba9afc079ca0ad0457f 2024-11-09T17:27:11,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/3f50f514c40f43858df6d3a532d47e4c 2024-11-09T17:27:11,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/00ae9e39e2724dfeb2b4e326d26db863 2024-11-09T17:27:11,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/63dc6879e07c47839bb628a8bd117b3b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d2dea898a5914d17acefe7d2e2d12e37, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d37c6c53f2f547d29bc0527e0174dbcc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/b4f735037cde4b19a617e9a4a78fc59d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/efc2eb94469a4d4d8c7a05b1dd3d64f5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9765b8fd426b442aae5a4650f4b67948, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/50e1d40babed4510a48f361e477c19f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c] to archive 2024-11-09T17:27:11,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:11,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/c0c1a672d6e7421ca025892b75c544d4 2024-11-09T17:27:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/77080da8ce5a4021a9d93f4d998b5bdb 2024-11-09T17:27:11,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/f65112cb04e54b1ebaad383683b3ec2a 2024-11-09T17:27:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/63dc6879e07c47839bb628a8bd117b3b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/63dc6879e07c47839bb628a8bd117b3b 2024-11-09T17:27:11,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/2d43b569892d4a0aa0522325fbf93964 2024-11-09T17:27:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/56d280e264bc49a2854b2fb201c13330 2024-11-09T17:27:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d2dea898a5914d17acefe7d2e2d12e37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d2dea898a5914d17acefe7d2e2d12e37 2024-11-09T17:27:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d8ae96863df341d1843c5e14f3384433 2024-11-09T17:27:11,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/25d2348a1b8f4b45b7abd15e5999b23f 2024-11-09T17:27:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d37c6c53f2f547d29bc0527e0174dbcc to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d37c6c53f2f547d29bc0527e0174dbcc 2024-11-09T17:27:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/52ade52a99c249b485089af7885c1fa4 2024-11-09T17:27:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/fcb2f915d8b44cefb7e40f7616838228 2024-11-09T17:27:11,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/b4f735037cde4b19a617e9a4a78fc59d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/b4f735037cde4b19a617e9a4a78fc59d 2024-11-09T17:27:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/0645ad6072d9437d9aa1849ee89c1747 2024-11-09T17:27:11,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/d84cdb953ea84ac18ebdd478fc411203 2024-11-09T17:27:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1fbdfd66d7c44eff8ea896a4e9d03941 2024-11-09T17:27:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/efc2eb94469a4d4d8c7a05b1dd3d64f5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/efc2eb94469a4d4d8c7a05b1dd3d64f5 2024-11-09T17:27:11,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/6540e9db266043718f014caf34549e17 2024-11-09T17:27:11,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f21e82c1d03411c949855640ec130a7 2024-11-09T17:27:11,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9765b8fd426b442aae5a4650f4b67948 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9765b8fd426b442aae5a4650f4b67948 2024-11-09T17:27:11,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/1f159951db574144962ef8c84b2c219f 2024-11-09T17:27:11,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/26fb9cbad15c4896adf0287a144094d7 2024-11-09T17:27:11,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/50e1d40babed4510a48f361e477c19f4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/50e1d40babed4510a48f361e477c19f4 2024-11-09T17:27:11,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/38686951d8db416685a1fd886be4616d 2024-11-09T17:27:11,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/9f20867898e946be8a623f6de84d0d1c 2024-11-09T17:27:11,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/3c4c32432438424ea47348f160d4f98c 2024-11-09T17:27:11,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2187dac00a0c431fae75156201283392, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cf4b8e1933214feeb9307e33926504c4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/ed8e6b4549d7488a9602ac6d99e10aff, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4f84b163494c4d949fba242bb37df5e2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/7f3b6ec830c946938794f14640d185e8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2156ab823ed64efe924e24203e294964, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2] to archive 2024-11-09T17:27:11,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:11,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cef592a1fa4a4f76b3443b22124b2af3 2024-11-09T17:27:11,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d4c24008132249bdafe7ea7a3a17fa69 2024-11-09T17:27:11,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3bdd4bfe53e14ec8a63dd1fdb2a9e2af 2024-11-09T17:27:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2187dac00a0c431fae75156201283392 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2187dac00a0c431fae75156201283392 2024-11-09T17:27:11,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/f3466964c07c4f16ab269b1c97a7b73c 2024-11-09T17:27:11,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/c4c710ae2b65475ead483bba0c622120 2024-11-09T17:27:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/268c9c74d1d547bab8c3a85e2d81029c 2024-11-09T17:27:11,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cf4b8e1933214feeb9307e33926504c4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cf4b8e1933214feeb9307e33926504c4 2024-11-09T17:27:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/60b52dcf6f8c45a68fc1bd2611011c8f 2024-11-09T17:27:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/50fb3b21de7c40088385ad292afe7ed1 2024-11-09T17:27:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/a22f435ecaeb4cbfbb800dfec2b2bc89 2024-11-09T17:27:11,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/ed8e6b4549d7488a9602ac6d99e10aff to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/ed8e6b4549d7488a9602ac6d99e10aff 2024-11-09T17:27:11,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/fb590ca0779643219b15338e1cc509b0 2024-11-09T17:27:11,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/04374a7d959c40ffa0257e04277fac8a 2024-11-09T17:27:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/cb27fb6487824910825e0feaf1ef7dbd 2024-11-09T17:27:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4f84b163494c4d949fba242bb37df5e2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4f84b163494c4d949fba242bb37df5e2 2024-11-09T17:27:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/3c06a634c3914a41b0b213092eb1711f 2024-11-09T17:27:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4c53926fb3534795991e35f3b8a1c836 2024-11-09T17:27:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/7f3b6ec830c946938794f14640d185e8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/7f3b6ec830c946938794f14640d185e8 2024-11-09T17:27:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/4e81ebd16baf44db83d9aab8586ff80f 2024-11-09T17:27:11,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/6338aa7004f34e9ebb8fbb8acc30363b 2024-11-09T17:27:11,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2156ab823ed64efe924e24203e294964 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/2156ab823ed64efe924e24203e294964 2024-11-09T17:27:11,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/d3d6bd8ed6d541b987b1114ef88d2af8 2024-11-09T17:27:11,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/67076fee7e1f414598518a02e7e76ba6 2024-11-09T17:27:11,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/404d2ec32c8f4c7f8e69e77637bd53f2 2024-11-09T17:27:11,998 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/recovered.edits/384.seqid, newMaxSeqId=384, maxSeqId=1 2024-11-09T17:27:11,998 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726. 2024-11-09T17:27:11,998 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] regionserver.HRegion(1635): Region close journal for bbbe0e290bd0d6582b636c1e26d5e726: 2024-11-09T17:27:11,999 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=92}] handler.UnassignRegionHandler(170): Closed bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:12,000 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=91 updating hbase:meta row=bbbe0e290bd0d6582b636c1e26d5e726, regionState=CLOSED 2024-11-09T17:27:12,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T17:27:12,001 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; CloseRegionProcedure bbbe0e290bd0d6582b636c1e26d5e726, server=c28fd47d982f,45633,1731173138129 in 1.5090 sec 2024-11-09T17:27:12,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=91, resume processing ppid=90 2024-11-09T17:27:12,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, ppid=90, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=bbbe0e290bd0d6582b636c1e26d5e726, UNASSIGN in 1.5120 sec 2024-11-09T17:27:12,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-11-09T17:27:12,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5140 sec 2024-11-09T17:27:12,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173232004"}]},"ts":"1731173232004"} 2024-11-09T17:27:12,005 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:27:12,045 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:27:12,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.6150 sec 2024-11-09T17:27:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-09T17:27:12,544 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 89 completed 2024-11-09T17:27:12,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:27:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,550 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=93, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,551 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=93, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-09T17:27:12,554 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:12,558 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/recovered.edits] 2024-11-09T17:27:12,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84ab8583b09a4c9eab460a24695fbd5c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/84ab8583b09a4c9eab460a24695fbd5c 2024-11-09T17:27:12,564 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/c39081c5bbcd44648eef59d41636245a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/A/c39081c5bbcd44648eef59d41636245a 2024-11-09T17:27:12,568 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/337c8803373b4610b6f0be6593525dd1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/337c8803373b4610b6f0be6593525dd1 2024-11-09T17:27:12,570 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/544245b1e993475cbfb783b86fe1bd89 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/B/544245b1e993475cbfb783b86fe1bd89 2024-11-09T17:27:12,575 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/44e269f1eda14fc6b3fa5a2f9995c84a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/44e269f1eda14fc6b3fa5a2f9995c84a 2024-11-09T17:27:12,578 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/5da55c9d7cb741f0a3fec9e21291f336 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/C/5da55c9d7cb741f0a3fec9e21291f336 2024-11-09T17:27:12,584 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/recovered.edits/384.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726/recovered.edits/384.seqid 2024-11-09T17:27:12,585 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/bbbe0e290bd0d6582b636c1e26d5e726 2024-11-09T17:27:12,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:27:12,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=93, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,597 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:27:12,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:27:12,600 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=93, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,600 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:27:12,600 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173232600"}]},"ts":"9223372036854775807"} 2024-11-09T17:27:12,602 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:27:12,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => bbbe0e290bd0d6582b636c1e26d5e726, NAME => 'TestAcidGuarantees,,1731173203609.bbbe0e290bd0d6582b636c1e26d5e726.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:27:12,602 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:27:12,602 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173232602"}]},"ts":"9223372036854775807"} 2024-11-09T17:27:12,603 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:27:12,646 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=93, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=93, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-09T17:27:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-09T17:27:12,653 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 93 completed 2024-11-09T17:27:12,664 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=457 (was 465), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 298) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5893 (was 5936) 2024-11-09T17:27:12,673 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobScanAtomicity Thread=238, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=5892 2024-11-09T17:27:12,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:27:12,675 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:27:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:12,677 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:27:12,677 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:12,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 94 2024-11-09T17:27:12,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:27:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:12,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742124_1300 (size=960) 2024-11-09T17:27:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:13,091 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:27:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742125_1301 (size=53) 2024-11-09T17:27:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:13,504 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:27:13,504 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing b5327c23ff0d8e9e58fbdf61393b5182, disabling compactions & flushes 2024-11-09T17:27:13,504 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. after waiting 0 ms 2024-11-09T17:27:13,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,505 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,505 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:13,507 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:27:13,508 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173233507"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173233507"}]},"ts":"1731173233507"} 2024-11-09T17:27:13,510 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:27:13,511 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:27:13,511 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173233511"}]},"ts":"1731173233511"} 2024-11-09T17:27:13,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:27:13,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, ASSIGN}] 2024-11-09T17:27:13,539 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, ASSIGN 2024-11-09T17:27:13,541 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=95, ppid=94, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:27:13,692 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=95 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:13,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; OpenRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:13,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:13,848 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:13,855 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,855 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7285): Opening region: {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:27:13,856 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,856 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:27:13,856 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7327): checking encryption for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,856 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7330): checking classloading for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,858 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,860 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:13,860 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName A 2024-11-09T17:27:13,861 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:13,861 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:13,861 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,863 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:13,863 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName B 2024-11-09T17:27:13,863 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:13,864 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:13,864 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,865 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:13,865 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName C 2024-11-09T17:27:13,866 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:13,866 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:13,866 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,867 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,868 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,870 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:27:13,872 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1085): writing seq id for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:13,875 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:27:13,875 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1102): Opened b5327c23ff0d8e9e58fbdf61393b5182; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58843764, jitterRate=-0.12315958738327026}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:27:13,876 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1001): Region open journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:13,877 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., pid=96, masterSystemTime=1731173233848 2024-11-09T17:27:13,878 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,878 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:13,879 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=95 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:13,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T17:27:13,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; OpenRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 in 186 msec 2024-11-09T17:27:13,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=95, resume processing ppid=94 2024-11-09T17:27:13,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, ppid=94, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, ASSIGN in 345 msec 2024-11-09T17:27:13,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:27:13,885 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173233885"}]},"ts":"1731173233885"} 2024-11-09T17:27:13,887 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:27:13,929 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=94, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:27:13,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=94, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.2540 sec 2024-11-09T17:27:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-09T17:27:14,790 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 94 completed 2024-11-09T17:27:14,794 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b82ba2a to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3637e4c6 2024-11-09T17:27:14,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f7d511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:14,842 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:14,845 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49996, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:14,847 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:27:14,847 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36512, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:27:14,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:27:14,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:27:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:14,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742126_1302 (size=996) 2024-11-09T17:27:15,284 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-09T17:27:15,284 INFO [PEWorker-5 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-09T17:27:15,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:27:15,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, REOPEN/MOVE}] 2024-11-09T17:27:15,288 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, REOPEN/MOVE 2024-11-09T17:27:15,289 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,290 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:27:15,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; CloseRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:15,441 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,441 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(124): Close b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:27:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1681): Closing b5327c23ff0d8e9e58fbdf61393b5182, disabling compactions & flushes 2024-11-09T17:27:15,441 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. after waiting 0 ms 2024-11-09T17:27:15,441 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,446 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T17:27:15,447 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,447 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegion(1635): Region close journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:15,447 WARN [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] regionserver.HRegionServer(3786): Not adding moved region record: b5327c23ff0d8e9e58fbdf61393b5182 to self. 2024-11-09T17:27:15,449 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=100}] handler.UnassignRegionHandler(170): Closed b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,450 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=CLOSED 2024-11-09T17:27:15,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T17:27:15,453 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; CloseRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 in 161 msec 2024-11-09T17:27:15,454 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=99, ppid=98, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, REOPEN/MOVE; state=CLOSED, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=true 2024-11-09T17:27:15,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=101, ppid=99, state=RUNNABLE; OpenRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:15,761 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,769 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,769 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7285): Opening region: {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:27:15,771 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,771 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:27:15,771 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7327): checking encryption for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,771 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(7330): checking classloading for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,774 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,775 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:15,775 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName A 2024-11-09T17:27:15,777 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:15,778 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:15,778 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,779 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:15,779 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName B 2024-11-09T17:27:15,780 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:15,780 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:15,780 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,781 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:15,781 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5327c23ff0d8e9e58fbdf61393b5182 columnFamilyName C 2024-11-09T17:27:15,781 DEBUG [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:15,782 INFO [StoreOpener-b5327c23ff0d8e9e58fbdf61393b5182-1 {}] regionserver.HStore(327): Store=b5327c23ff0d8e9e58fbdf61393b5182/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:15,782 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,783 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,785 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,786 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:27:15,788 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1085): writing seq id for b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,789 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1102): Opened b5327c23ff0d8e9e58fbdf61393b5182; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72522589, jitterRate=0.0806707888841629}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:27:15,790 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegion(1001): Region open journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:15,790 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., pid=101, masterSystemTime=1731173235761 2024-11-09T17:27:15,792 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,792 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=101}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:15,792 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=99 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=OPEN, openSeqNum=5, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=101, resume processing ppid=99 2024-11-09T17:27:15,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=101, ppid=99, state=SUCCESS; OpenRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 in 185 msec 2024-11-09T17:27:15,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=99, resume processing ppid=98 2024-11-09T17:27:15,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, ppid=98, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, REOPEN/MOVE in 506 msec 2024-11-09T17:27:15,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-11-09T17:27:15,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-09T17:27:15,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 948 msec 2024-11-09T17:27:15,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-09T17:27:15,802 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b6cf8cb to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72f422b4 2024-11-09T17:27:15,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc42ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,869 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ec15031 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2df33cdf 2024-11-09T17:27:15,879 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117e86d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,880 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3dd5b441 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9f472e0 2024-11-09T17:27:15,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cd96549, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,888 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c336ea4 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@167a78b0 2024-11-09T17:27:15,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31aea41b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,896 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f94d721 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5aee939b 2024-11-09T17:27:15,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e247aa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,905 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f49665c to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2205f666 2024-11-09T17:27:15,913 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27539bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,913 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x683f8469 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6584e9ce 2024-11-09T17:27:15,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e3203d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,922 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75e4d3d0 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37ec8e3b 2024-11-09T17:27:15,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798e7fd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,931 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b308f62 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@787e5169 2024-11-09T17:27:15,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7284f16d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,938 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68035c67 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@627cad17 2024-11-09T17:27:15,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a637ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:15,948 DEBUG [hconnection-0x1d6bb5cc-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,948 DEBUG [hconnection-0x3779370a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,948 DEBUG [hconnection-0x4ddf47fa-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,949 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50004, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,949 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50006, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,949 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50020, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,950 DEBUG [hconnection-0x4963c24-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,951 DEBUG [hconnection-0x20198090-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,951 DEBUG [hconnection-0x3bcc6fa7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,951 DEBUG [hconnection-0x56c308d3-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:15,951 DEBUG [hconnection-0x5a4c499e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,952 DEBUG [hconnection-0x7c513998-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees 2024-11-09T17:27:15,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:15,953 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50044, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,953 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50052, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,953 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50088, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,953 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50072, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,954 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=102, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:15,955 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50034, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,955 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=102, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:15,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:15,955 DEBUG [hconnection-0x3a44c5d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:15,955 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50056, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,957 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50098, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:15,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:15,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:15,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173295970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:15,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173295970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173295970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173295971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:15,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173295971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097c77f7fcda49432ba40958a1d54a07a2_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173235958/Put/seqid=0 2024-11-09T17:27:16,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742127_1303 (size=12154) 2024-11-09T17:27:16,004 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:16,009 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097c77f7fcda49432ba40958a1d54a07a2_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c77f7fcda49432ba40958a1d54a07a2_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:16,010 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/aba4dd79257f43b98be0b8afb8d9d61f, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:16,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/aba4dd79257f43b98be0b8afb8d9d61f is 175, key is test_row_0/A:col10/1731173235958/Put/seqid=0 2024-11-09T17:27:16,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742128_1304 (size=30955) 2024-11-09T17:27:16,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:16,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173296072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173296072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173296079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173296079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173296082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,107 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:16,260 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173296277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173296277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173296281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173296283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173296287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,412 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,422 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=18, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/aba4dd79257f43b98be0b8afb8d9d61f 2024-11-09T17:27:16,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2d72cd651f27440ab2c142f960367692 is 50, key is test_row_0/B:col10/1731173235958/Put/seqid=0 2024-11-09T17:27:16,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742129_1305 (size=12001) 2024-11-09T17:27:16,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2d72cd651f27440ab2c142f960367692 2024-11-09T17:27:16,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/d27b23d4249e42169b12a930142fa758 is 50, key is test_row_0/C:col10/1731173235958/Put/seqid=0 2024-11-09T17:27:16,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742130_1306 (size=12001) 2024-11-09T17:27:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:16,564 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173296581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173296584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173296588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173296589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173296595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,717 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,717 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,870 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:16,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/d27b23d4249e42169b12a930142fa758 2024-11-09T17:27:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:16,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:16,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] handler.RSProcedureHandler(58): pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=103 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=103 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:16,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/aba4dd79257f43b98be0b8afb8d9d61f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f 2024-11-09T17:27:16,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f, entries=150, sequenceid=18, filesize=30.2 K 2024-11-09T17:27:16,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2d72cd651f27440ab2c142f960367692 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692 2024-11-09T17:27:16,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692, entries=150, sequenceid=18, filesize=11.7 K 2024-11-09T17:27:16,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/d27b23d4249e42169b12a930142fa758 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758 2024-11-09T17:27:16,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758, entries=150, sequenceid=18, filesize=11.7 K 2024-11-09T17:27:16,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for b5327c23ff0d8e9e58fbdf61393b5182 in 926ms, sequenceid=18, compaction requested=false 2024-11-09T17:27:16,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:17,022 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=103 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:17,023 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:17,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110989ea38f76fcd46ccb1eceb7ea2d04473_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173235970/Put/seqid=0 2024-11-09T17:27:17,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742131_1307 (size=12154) 2024-11-09T17:27:17,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:17,045 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110989ea38f76fcd46ccb1eceb7ea2d04473_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989ea38f76fcd46ccb1eceb7ea2d04473_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/231ff27dff594d3395eda798325e427d, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/231ff27dff594d3395eda798325e427d is 175, key is test_row_0/A:col10/1731173235970/Put/seqid=0 2024-11-09T17:27:17,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742132_1308 (size=30955) 2024-11-09T17:27:17,049 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/231ff27dff594d3395eda798325e427d 2024-11-09T17:27:17,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a127d47a2e8648c4b25f08c553c8565d is 50, key is test_row_0/B:col10/1731173235970/Put/seqid=0 2024-11-09T17:27:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742133_1309 (size=12001) 2024-11-09T17:27:17,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:17,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:17,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173297134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173297135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173297135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173297136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173297137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173297238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173297243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173297243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173297244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173297244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173297442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173297451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173297451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173297451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173297452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,459 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a127d47a2e8648c4b25f08c553c8565d 2024-11-09T17:27:17,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/b076f38b01d3468581da233d74a97cb4 is 50, key is test_row_0/C:col10/1731173235970/Put/seqid=0 2024-11-09T17:27:17,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742134_1310 (size=12001) 2024-11-09T17:27:17,478 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/b076f38b01d3468581da233d74a97cb4 2024-11-09T17:27:17,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/231ff27dff594d3395eda798325e427d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d 2024-11-09T17:27:17,486 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d, entries=150, sequenceid=41, filesize=30.2 K 2024-11-09T17:27:17,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a127d47a2e8648c4b25f08c553c8565d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d 2024-11-09T17:27:17,490 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T17:27:17,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/b076f38b01d3468581da233d74a97cb4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4 2024-11-09T17:27:17,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4, entries=150, sequenceid=41, filesize=11.7 K 2024-11-09T17:27:17,495 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for b5327c23ff0d8e9e58fbdf61393b5182 in 471ms, sequenceid=41, compaction requested=false 2024-11-09T17:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:17,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=103}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=103 2024-11-09T17:27:17,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=103 2024-11-09T17:27:17,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=103, resume processing ppid=102 2024-11-09T17:27:17,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=103, ppid=102, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5410 sec 2024-11-09T17:27:17,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=102, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=102, table=TestAcidGuarantees in 1.5460 sec 2024-11-09T17:27:17,522 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:27:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:17,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:27:17,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:17,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:17,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fc1d73696804157a2ed44b9b6ba9697_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:17,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742135_1311 (size=14594) 2024-11-09T17:27:17,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173297771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173297772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173297778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173297779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173297779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173297880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173297882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173297884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173297890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:17,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:17,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173297890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-09T17:27:18,056 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 102 completed 2024-11-09T17:27:18,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees 2024-11-09T17:27:18,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=104, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:18,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=104, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:18,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:18,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173298086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173298088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173298089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173298094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173298094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:18,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:18,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094fc1d73696804157a2ed44b9b6ba9697_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc1d73696804157a2ed44b9b6ba9697_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:18,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7fe52d55113b4961bf0138554a928a94, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:18,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7fe52d55113b4961bf0138554a928a94 is 175, key is test_row_0/A:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:18,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742136_1312 (size=39549) 2024-11-09T17:27:18,210 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:18,362 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173298391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173298392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173298393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173298399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173298401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,515 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7fe52d55113b4961bf0138554a928a94 2024-11-09T17:27:18,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/d21559e5c7f043508c27fd82b890a58e is 50, key is test_row_0/B:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:18,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742137_1313 (size=12001) 2024-11-09T17:27:18,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:18,667 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,819 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173298898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173298900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173298901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173298904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173298905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,971 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:18,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:18,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:18,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/d21559e5c7f043508c27fd82b890a58e 2024-11-09T17:27:18,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/8e14e3958c664396897b34ae5c27dfa9 is 50, key is test_row_0/C:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:18,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742138_1314 (size=12001) 2024-11-09T17:27:19,124 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:19,276 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:19,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:19,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] handler.RSProcedureHandler(58): pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=105 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=105 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:19,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/8e14e3958c664396897b34ae5c27dfa9 2024-11-09T17:27:19,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7fe52d55113b4961bf0138554a928a94 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94 2024-11-09T17:27:19,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94, entries=200, sequenceid=56, filesize=38.6 K 2024-11-09T17:27:19,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/d21559e5c7f043508c27fd82b890a58e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e 2024-11-09T17:27:19,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T17:27:19,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/8e14e3958c664396897b34ae5c27dfa9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9 2024-11-09T17:27:19,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T17:27:19,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b5327c23ff0d8e9e58fbdf61393b5182 in 1655ms, sequenceid=56, compaction requested=true 2024-11-09T17:27:19,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:19,402 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:19,402 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:19,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:19,402 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:19,402 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:19,403 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,403 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,403 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=99.1 K 2024-11-09T17:27:19,403 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.2 K 2024-11-09T17:27:19,403 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94] 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2d72cd651f27440ab2c142f960367692, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731173235958 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting aba4dd79257f43b98be0b8afb8d9d61f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731173235958 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a127d47a2e8648c4b25f08c553c8565d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173235969 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 231ff27dff594d3395eda798325e427d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173235969 2024-11-09T17:27:19,403 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d21559e5c7f043508c27fd82b890a58e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:19,404 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7fe52d55113b4961bf0138554a928a94, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:19,408 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:19,411 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:19,411 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/c04512ca14494d21800c3620b26e20cd is 50, key is test_row_0/B:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:19,412 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109cca26ab575a74873b90eabb2a4180c75_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:19,413 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109cca26ab575a74873b90eabb2a4180c75_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:19,413 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cca26ab575a74873b90eabb2a4180c75_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:19,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742139_1315 (size=12104) 2024-11-09T17:27:19,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742140_1316 (size=4469) 2024-11-09T17:27:19,428 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=105 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,429 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:19,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110993faa00fc34543b7b9fd773a0acbce41_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173237776/Put/seqid=0 2024-11-09T17:27:19,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742141_1317 (size=12154) 2024-11-09T17:27:19,819 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/c04512ca14494d21800c3620b26e20cd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/c04512ca14494d21800c3620b26e20cd 2024-11-09T17:27:19,822 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#266 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:19,822 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/b1d88e49666948c19410354cd79c8006 is 175, key is test_row_0/A:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:19,822 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into c04512ca14494d21800c3620b26e20cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:19,822 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:19,822 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=13, startTime=1731173239402; duration=0sec 2024-11-09T17:27:19,822 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:19,822 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:19,822 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:19,824 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:19,824 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:19,824 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:19,824 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.2 K 2024-11-09T17:27:19,824 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d27b23d4249e42169b12a930142fa758, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=18, earliestPutTs=1731173235958 2024-11-09T17:27:19,824 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting b076f38b01d3468581da233d74a97cb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731173235969 2024-11-09T17:27:19,827 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 8e14e3958c664396897b34ae5c27dfa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:19,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742142_1318 (size=31058) 2024-11-09T17:27:19,833 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#269 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:19,833 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/2e4d1d642b4749838cb11f961b8315db is 50, key is test_row_0/C:col10/1731173237134/Put/seqid=0 2024-11-09T17:27:19,838 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/b1d88e49666948c19410354cd79c8006 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006 2024-11-09T17:27:19,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742143_1319 (size=12104) 2024-11-09T17:27:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:19,844 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into b1d88e49666948c19410354cd79c8006(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:19,844 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:19,844 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=13, startTime=1731173239401; duration=0sec 2024-11-09T17:27:19,844 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:19,844 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:19,844 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110993faa00fc34543b7b9fd773a0acbce41_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110993faa00fc34543b7b9fd773a0acbce41_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:19,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/cc03eac6f9f84eb396f96c979b4a7eb9, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/cc03eac6f9f84eb396f96c979b4a7eb9 is 175, key is test_row_0/A:col10/1731173237776/Put/seqid=0 2024-11-09T17:27:19,846 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/2e4d1d642b4749838cb11f961b8315db as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2e4d1d642b4749838cb11f961b8315db 2024-11-09T17:27:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742144_1320 (size=30955) 2024-11-09T17:27:19,853 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into 2e4d1d642b4749838cb11f961b8315db(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:19,854 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:19,854 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=13, startTime=1731173239402; duration=0sec 2024-11-09T17:27:19,854 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:19,854 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:19,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:19,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173299917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173299918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173299920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173299920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:19,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:19,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173299921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173300024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173300024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173300024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173300026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:20,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173300227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173300228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173300229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173300229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/cc03eac6f9f84eb396f96c979b4a7eb9 2024-11-09T17:27:20,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/40d6920439b24f0fb871a1a70d9c8007 is 50, key is test_row_0/B:col10/1731173237776/Put/seqid=0 2024-11-09T17:27:20,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742145_1321 (size=12001) 2024-11-09T17:27:20,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173300532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173300532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173300532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173300533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:20,665 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/40d6920439b24f0fb871a1a70d9c8007 2024-11-09T17:27:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/f3e379021d5145fa8e10a88dc7d4ff30 is 50, key is test_row_0/C:col10/1731173237776/Put/seqid=0 2024-11-09T17:27:20,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742146_1322 (size=12001) 2024-11-09T17:27:21,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:21,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173301037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173301038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:21,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:21,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173301040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:21,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173301041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:21,075 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/f3e379021d5145fa8e10a88dc7d4ff30 2024-11-09T17:27:21,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/cc03eac6f9f84eb396f96c979b4a7eb9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9 2024-11-09T17:27:21,123 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9, entries=150, sequenceid=77, filesize=30.2 K 2024-11-09T17:27:21,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/40d6920439b24f0fb871a1a70d9c8007 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007 2024-11-09T17:27:21,127 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T17:27:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/f3e379021d5145fa8e10a88dc7d4ff30 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30 2024-11-09T17:27:21,130 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30, entries=150, sequenceid=77, filesize=11.7 K 2024-11-09T17:27:21,131 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b5327c23ff0d8e9e58fbdf61393b5182 in 1702ms, sequenceid=77, compaction requested=false 2024-11-09T17:27:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-11-09T17:27:21,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=105 2024-11-09T17:27:21,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=105, resume processing ppid=104 2024-11-09T17:27:21,133 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=105, ppid=104, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0730 sec 2024-11-09T17:27:21,134 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=104, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=104, table=TestAcidGuarantees in 3.0760 sec 2024-11-09T17:27:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:21,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:21,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:21,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091b301dd617474eecacc73b8fa4fc16cd_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:21,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742147_1323 (size=14594) 2024-11-09T17:27:22,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173302010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173302044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173302047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173302047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173302052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173302116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-09T17:27:22,162 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 104 completed 2024-11-09T17:27:22,163 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees 2024-11-09T17:27:22,164 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:22,165 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=106, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:22,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:22,316 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:22,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173302325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,345 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:22,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091b301dd617474eecacc73b8fa4fc16cd_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b301dd617474eecacc73b8fa4fc16cd_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:22,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/e6ddfd1a1b06407aab1c7639afcf9c05, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:22,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/e6ddfd1a1b06407aab1c7639afcf9c05 is 175, key is test_row_0/A:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:22,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742148_1324 (size=39549) 2024-11-09T17:27:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:22,468 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,620 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:22,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173302632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,753 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/e6ddfd1a1b06407aab1c7639afcf9c05 2024-11-09T17:27:22,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/672fba0735654b6eae228dfc64561680 is 50, key is test_row_0/B:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742149_1325 (size=12001) 2024-11-09T17:27:22,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:22,772 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,925 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:22,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:22,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:23,077 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:23,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] handler.RSProcedureHandler(58): pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=107 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:23,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=107 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:23,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:23,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173303138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:23,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/672fba0735654b6eae228dfc64561680 2024-11-09T17:27:23,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0c830365083b4b84a34ad029e69b213f is 50, key is test_row_0/C:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:23,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742150_1326 (size=12001) 2024-11-09T17:27:23,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0c830365083b4b84a34ad029e69b213f 2024-11-09T17:27:23,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/e6ddfd1a1b06407aab1c7639afcf9c05 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05 2024-11-09T17:27:23,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05, entries=200, sequenceid=96, filesize=38.6 K 2024-11-09T17:27:23,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/672fba0735654b6eae228dfc64561680 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680 2024-11-09T17:27:23,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T17:27:23,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0c830365083b4b84a34ad029e69b213f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f 2024-11-09T17:27:23,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-09T17:27:23,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b5327c23ff0d8e9e58fbdf61393b5182 in 1253ms, sequenceid=96, compaction requested=true 2024-11-09T17:27:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:23,188 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:23,188 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:23,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:23,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:23,189 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:23,189 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:23,189 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:23,189 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:23,189 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,189 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,189 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=99.2 K 2024-11-09T17:27:23,189 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/c04512ca14494d21800c3620b26e20cd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.3 K 2024-11-09T17:27:23,189 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,189 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05] 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c04512ca14494d21800c3620b26e20cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b1d88e49666948c19410354cd79c8006, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 40d6920439b24f0fb871a1a70d9c8007, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173237776 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cc03eac6f9f84eb396f96c979b4a7eb9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173237776 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 672fba0735654b6eae228dfc64561680, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239919 2024-11-09T17:27:23,190 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting e6ddfd1a1b06407aab1c7639afcf9c05, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239917 2024-11-09T17:27:23,197 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:23,198 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#275 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:23,199 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9cc7cea3309f4d2598ba220e9f4c2ba3 is 50, key is test_row_0/B:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:23,203 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110976b3078f23624735a3c10bb6168cbd4a_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:23,205 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110976b3078f23624735a3c10bb6168cbd4a_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:23,205 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976b3078f23624735a3c10bb6168cbd4a_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:23,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742151_1327 (size=12207) 2024-11-09T17:27:23,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742152_1328 (size=4469) 2024-11-09T17:27:23,229 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:23,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=107 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,230 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:23,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fd184d0b0bf4120b428dd2f6634fd55_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173242004/Put/seqid=0 2024-11-09T17:27:23,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742153_1329 (size=12154) 2024-11-09T17:27:23,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:23,615 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#276 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:23,616 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3f95d9b19de141c9a88fc972e5402dc4 is 175, key is test_row_0/A:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:23,619 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9cc7cea3309f4d2598ba220e9f4c2ba3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cc7cea3309f4d2598ba220e9f4c2ba3 2024-11-09T17:27:23,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742154_1330 (size=31161) 2024-11-09T17:27:23,623 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into 9cc7cea3309f4d2598ba220e9f4c2ba3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:23,623 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:23,623 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=13, startTime=1731173243188; duration=0sec 2024-11-09T17:27:23,623 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:23,623 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:23,623 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:23,624 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3f95d9b19de141c9a88fc972e5402dc4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4 2024-11-09T17:27:23,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:23,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:23,625 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:23,625 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2e4d1d642b4749838cb11f961b8315db, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.3 K 2024-11-09T17:27:23,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2e4d1d642b4749838cb11f961b8315db, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173237134 2024-11-09T17:27:23,626 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f3e379021d5145fa8e10a88dc7d4ff30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731173237776 2024-11-09T17:27:23,626 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0c830365083b4b84a34ad029e69b213f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239919 2024-11-09T17:27:23,629 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into 3f95d9b19de141c9a88fc972e5402dc4(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:23,629 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:23,629 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=13, startTime=1731173243188; duration=0sec 2024-11-09T17:27:23,629 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:23,629 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:23,632 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:23,632 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/ffab9f795c7a4d33a1ee3ba287cd4fd8 is 50, key is test_row_0/C:col10/1731173239920/Put/seqid=0 2024-11-09T17:27:23,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742155_1331 (size=12207) 2024-11-09T17:27:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:23,643 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099fd184d0b0bf4120b428dd2f6634fd55_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fd184d0b0bf4120b428dd2f6634fd55_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/2c497b5a7cc34d2ebb9629d03c92cfc7, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/2c497b5a7cc34d2ebb9629d03c92cfc7 is 175, key is test_row_0/A:col10/1731173242004/Put/seqid=0 2024-11-09T17:27:23,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742156_1332 (size=30955) 2024-11-09T17:27:24,041 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/ffab9f795c7a4d33a1ee3ba287cd4fd8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ffab9f795c7a4d33a1ee3ba287cd4fd8 2024-11-09T17:27:24,045 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into ffab9f795c7a4d33a1ee3ba287cd4fd8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:24,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:24,045 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=13, startTime=1731173243189; duration=0sec 2024-11-09T17:27:24,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:24,045 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:24,048 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/2c497b5a7cc34d2ebb9629d03c92cfc7 2024-11-09T17:27:24,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2db7b2fded644453bb088ba6bdab03b0 is 50, key is test_row_0/B:col10/1731173242004/Put/seqid=0 2024-11-09T17:27:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:24,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:24,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742157_1333 (size=12001) 2024-11-09T17:27:24,064 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2db7b2fded644453bb088ba6bdab03b0 2024-11-09T17:27:24,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0aa1f0271ea045cab335d56b4f0d51e1 is 50, key is test_row_0/C:col10/1731173242004/Put/seqid=0 2024-11-09T17:27:24,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742158_1334 (size=12001) 2024-11-09T17:27:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173304083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173304084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173304084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173304085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173304142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173304186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173304191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173304191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173304191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:24,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173304390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173304396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173304397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173304397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,477 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0aa1f0271ea045cab335d56b4f0d51e1 2024-11-09T17:27:24,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/2c497b5a7cc34d2ebb9629d03c92cfc7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7 2024-11-09T17:27:24,484 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7, entries=150, sequenceid=116, filesize=30.2 K 2024-11-09T17:27:24,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/2db7b2fded644453bb088ba6bdab03b0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0 2024-11-09T17:27:24,489 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:27:24,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0aa1f0271ea045cab335d56b4f0d51e1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1 2024-11-09T17:27:24,493 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:27:24,494 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b5327c23ff0d8e9e58fbdf61393b5182 in 1264ms, sequenceid=116, compaction requested=false 2024-11-09T17:27:24,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:24,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:24,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-11-09T17:27:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-11-09T17:27:24,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-11-09T17:27:24,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-09T17:27:24,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=106, table=TestAcidGuarantees in 2.3340 sec 2024-11-09T17:27:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:24,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:24,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:24,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097126030293d54bca970c18e8d8887e6a_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:24,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742159_1335 (size=14794) 2024-11-09T17:27:24,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173304722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173304723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173304725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173304726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173304828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173304829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173304829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173304833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173305033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173305033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173305035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173305039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,108 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:25,111 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097126030293d54bca970c18e8d8887e6a_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097126030293d54bca970c18e8d8887e6a_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:25,112 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/9141ae54f50d47e082aedaf745c4b66f, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:25,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/9141ae54f50d47e082aedaf745c4b66f is 175, key is test_row_0/A:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:25,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742160_1336 (size=39749) 2024-11-09T17:27:25,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173305336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173305337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173305337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173305344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/9141ae54f50d47e082aedaf745c4b66f 2024-11-09T17:27:25,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/7f0607fea8a54312a642afa9aa452931 is 50, key is test_row_0/B:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:25,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742161_1337 (size=12151) 2024-11-09T17:27:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173305847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173305849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173305849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173305853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:25,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/7f0607fea8a54312a642afa9aa452931 2024-11-09T17:27:25,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/c403f2ab8b8e441a84e81429c4449b3b is 50, key is test_row_0/C:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:25,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742162_1338 (size=12151) 2024-11-09T17:27:25,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/c403f2ab8b8e441a84e81429c4449b3b 2024-11-09T17:27:25,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/9141ae54f50d47e082aedaf745c4b66f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f 2024-11-09T17:27:25,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f, entries=200, sequenceid=136, filesize=38.8 K 2024-11-09T17:27:25,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/7f0607fea8a54312a642afa9aa452931 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931 2024-11-09T17:27:25,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931, entries=150, sequenceid=136, filesize=11.9 K 2024-11-09T17:27:25,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/c403f2ab8b8e441a84e81429c4449b3b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b 2024-11-09T17:27:25,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b, entries=150, sequenceid=136, filesize=11.9 K 2024-11-09T17:27:25,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b5327c23ff0d8e9e58fbdf61393b5182 in 1264ms, sequenceid=136, compaction requested=true 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:25,961 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:25,961 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:25,962 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:25,962 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:25,962 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:25,962 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:25,962 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:25,962 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:25,962 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cc7cea3309f4d2598ba220e9f4c2ba3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.5 K 2024-11-09T17:27:25,962 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=99.5 K 2024-11-09T17:27:25,962 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:25,962 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f] 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9cc7cea3309f4d2598ba220e9f4c2ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239919 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3f95d9b19de141c9a88fc972e5402dc4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239919 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2db7b2fded644453bb088ba6bdab03b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173241973 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2c497b5a7cc34d2ebb9629d03c92cfc7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173241973 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7f0607fea8a54312a642afa9aa452931, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244082 2024-11-09T17:27:25,963 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9141ae54f50d47e082aedaf745c4b66f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244081 2024-11-09T17:27:25,970 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:25,971 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:25,971 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/d9ba6372ae1c42e69554a1f17b85303c is 50, key is test_row_0/B:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:25,981 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ae8965560c774a778a7967be2b2db055_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:25,983 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ae8965560c774a778a7967be2b2db055_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:25,983 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ae8965560c774a778a7967be2b2db055_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:25,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742163_1339 (size=12459) 2024-11-09T17:27:25,990 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/d9ba6372ae1c42e69554a1f17b85303c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d9ba6372ae1c42e69554a1f17b85303c 2024-11-09T17:27:25,996 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into d9ba6372ae1c42e69554a1f17b85303c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:25,996 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:25,996 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=13, startTime=1731173245961; duration=0sec 2024-11-09T17:27:25,997 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:25,997 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:25,997 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:25,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742164_1340 (size=4469) 2024-11-09T17:27:25,999 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:25,999 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:25,999 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:25,999 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ffab9f795c7a4d33a1ee3ba287cd4fd8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=35.5 K 2024-11-09T17:27:25,999 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ffab9f795c7a4d33a1ee3ba287cd4fd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731173239919 2024-11-09T17:27:26,000 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0aa1f0271ea045cab335d56b4f0d51e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173241973 2024-11-09T17:27:26,000 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c403f2ab8b8e441a84e81429c4449b3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244082 2024-11-09T17:27:26,006 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#286 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:26,006 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/36e95ac3377f4ed694718822c1b9fd5c is 50, key is test_row_0/C:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:26,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742165_1341 (size=12459) 2024-11-09T17:27:26,014 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/36e95ac3377f4ed694718822c1b9fd5c as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/36e95ac3377f4ed694718822c1b9fd5c 2024-11-09T17:27:26,019 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into 36e95ac3377f4ed694718822c1b9fd5c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:26,019 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:26,019 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=13, startTime=1731173245961; duration=0sec 2024-11-09T17:27:26,019 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:26,019 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:26,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-09T17:27:26,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:26,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:26,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:26,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096c7645ca350e47528ecd31db9526c251_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173244721/Put/seqid=0 2024-11-09T17:27:26,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742166_1342 (size=12304) 2024-11-09T17:27:26,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173306240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-09T17:27:26,268 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 106 completed 2024-11-09T17:27:26,269 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees 2024-11-09T17:27:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:26,270 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:26,271 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=108, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:26,271 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173306344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:26,399 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#285 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:26,400 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/b4a9e62cadd54bb9b06363959be4bc0f is 175, key is test_row_0/A:col10/1731173244084/Put/seqid=0 2024-11-09T17:27:26,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742167_1343 (size=31413) 2024-11-09T17:27:26,422 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:26,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173306548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:26,574 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,602 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:26,605 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096c7645ca350e47528ecd31db9526c251_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096c7645ca350e47528ecd31db9526c251_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:26,606 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/434e439c9add457b910ab3c407bc3f1e, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:26,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/434e439c9add457b910ab3c407bc3f1e is 175, key is test_row_0/A:col10/1731173244721/Put/seqid=0 2024-11-09T17:27:26,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742168_1344 (size=31105) 2024-11-09T17:27:26,726 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,807 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/b4a9e62cadd54bb9b06363959be4bc0f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f 2024-11-09T17:27:26,811 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into b4a9e62cadd54bb9b06363959be4bc0f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:26,811 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:26,811 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=13, startTime=1731173245961; duration=0sec 2024-11-09T17:27:26,811 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:26,811 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173306855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173306856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173306857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173306858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:26,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173306863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:26,879 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:26,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:26,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:26,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,010 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/434e439c9add457b910ab3c407bc3f1e 2024-11-09T17:27:27,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9f58db871bdb4718b3b5af372d8e339e is 50, key is test_row_0/B:col10/1731173244721/Put/seqid=0 2024-11-09T17:27:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742169_1345 (size=12151) 2024-11-09T17:27:27,031 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,183 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,336 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173307363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:27,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9f58db871bdb4718b3b5af372d8e339e 2024-11-09T17:27:27,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0962eb067abc4629b0adefe4515a2fe9 is 50, key is test_row_0/C:col10/1731173244721/Put/seqid=0 2024-11-09T17:27:27,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742170_1346 (size=12151) 2024-11-09T17:27:27,488 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,640 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,793 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] handler.RSProcedureHandler(58): pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=109 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=109 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:27,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0962eb067abc4629b0adefe4515a2fe9 2024-11-09T17:27:27,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/434e439c9add457b910ab3c407bc3f1e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e 2024-11-09T17:27:27,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e, entries=150, sequenceid=158, filesize=30.4 K 2024-11-09T17:27:27,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9f58db871bdb4718b3b5af372d8e339e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e 2024-11-09T17:27:27,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e, entries=150, sequenceid=158, filesize=11.9 K 2024-11-09T17:27:27,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0962eb067abc4629b0adefe4515a2fe9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9 2024-11-09T17:27:27,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9, entries=150, sequenceid=158, filesize=11.9 K 2024-11-09T17:27:27,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b5327c23ff0d8e9e58fbdf61393b5182 in 1655ms, sequenceid=158, compaction requested=false 2024-11-09T17:27:27,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:27,945 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:27,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=109 2024-11-09T17:27:27,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:27,946 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:27,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:27,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110938673eca34a3495898b3a91e158e3dd2_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173246227/Put/seqid=0 2024-11-09T17:27:27,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742171_1347 (size=12304) 2024-11-09T17:27:28,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:28,359 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110938673eca34a3495898b3a91e158e3dd2_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938673eca34a3495898b3a91e158e3dd2_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/52b66510306047698d29be2e5bdbbba8, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:28,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/52b66510306047698d29be2e5bdbbba8 is 175, key is test_row_0/A:col10/1731173246227/Put/seqid=0 2024-11-09T17:27:28,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742172_1348 (size=31105) 2024-11-09T17:27:28,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:28,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:28,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173308432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173308537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173308744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,764 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/52b66510306047698d29be2e5bdbbba8 2024-11-09T17:27:28,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9cd7dd43cedc4af2b52ced4c718d81a1 is 50, key is test_row_0/B:col10/1731173246227/Put/seqid=0 2024-11-09T17:27:28,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742173_1349 (size=12151) 2024-11-09T17:27:28,779 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9cd7dd43cedc4af2b52ced4c718d81a1 2024-11-09T17:27:28,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/2637d48a98854ab698e1985ea8ed1d31 is 50, key is test_row_0/C:col10/1731173246227/Put/seqid=0 2024-11-09T17:27:28,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742174_1350 (size=12151) 2024-11-09T17:27:28,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173308862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,867 DEBUG [Thread-1399 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4144 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:28,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173308871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,874 DEBUG [Thread-1395 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4151 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173308874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173308875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:28,880 DEBUG [Thread-1397 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4153 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:28,880 DEBUG [Thread-1393 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4155 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:29,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173309048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:29,190 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/2637d48a98854ab698e1985ea8ed1d31 2024-11-09T17:27:29,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/52b66510306047698d29be2e5bdbbba8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8 2024-11-09T17:27:29,197 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8, entries=150, sequenceid=175, filesize=30.4 K 2024-11-09T17:27:29,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/9cd7dd43cedc4af2b52ced4c718d81a1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1 2024-11-09T17:27:29,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T17:27:29,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/2637d48a98854ab698e1985ea8ed1d31 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31 2024-11-09T17:27:29,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31, entries=150, sequenceid=175, filesize=11.9 K 2024-11-09T17:27:29,204 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b5327c23ff0d8e9e58fbdf61393b5182 in 1259ms, sequenceid=175, compaction requested=true 2024-11-09T17:27:29,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:29,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-11-09T17:27:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-11-09T17:27:29,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-11-09T17:27:29,206 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9340 sec 2024-11-09T17:27:29,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=108, table=TestAcidGuarantees in 2.9380 sec 2024-11-09T17:27:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:29,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:29,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:29,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e82141f585c4ff8ae4c102672ea1730_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:29,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742175_1351 (size=14794) 2024-11-09T17:27:29,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173309607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:29,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173309714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:29,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:29,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173309922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:29,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:29,971 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093e82141f585c4ff8ae4c102672ea1730_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e82141f585c4ff8ae4c102672ea1730_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:29,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/977ce91566514698997b504fd3fceeb3, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:29,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/977ce91566514698997b504fd3fceeb3 is 175, key is test_row_0/A:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:29,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742176_1352 (size=39749) 2024-11-09T17:27:30,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:30,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173310228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-09T17:27:30,374 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 108 completed 2024-11-09T17:27:30,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees 2024-11-09T17:27:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:30,376 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:30,377 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=110, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:30,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:30,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/977ce91566514698997b504fd3fceeb3 2024-11-09T17:27:30,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/f591f60575504e6cb6747d0005a28171 is 50, key is test_row_0/B:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:30,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742177_1353 (size=12151) 2024-11-09T17:27:30,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:30,528 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:30,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:30,680 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:30,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:30,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173310737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/f591f60575504e6cb6747d0005a28171 2024-11-09T17:27:30,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/88bdf5958b2e4f629b9a96b78820ab5a is 50, key is test_row_0/C:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:30,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742178_1354 (size=12151) 2024-11-09T17:27:30,833 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:30,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:30,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:30,985 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:30,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:30,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:30,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:30,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:31,137 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:31,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] handler.RSProcedureHandler(58): pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=111 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=111 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:31,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/88bdf5958b2e4f629b9a96b78820ab5a 2024-11-09T17:27:31,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/977ce91566514698997b504fd3fceeb3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3 2024-11-09T17:27:31,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3, entries=200, sequenceid=196, filesize=38.8 K 2024-11-09T17:27:31,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/f591f60575504e6cb6747d0005a28171 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171 2024-11-09T17:27:31,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T17:27:31,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/88bdf5958b2e4f629b9a96b78820ab5a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a 2024-11-09T17:27:31,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a, entries=150, sequenceid=196, filesize=11.9 K 2024-11-09T17:27:31,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b5327c23ff0d8e9e58fbdf61393b5182 in 1713ms, sequenceid=196, compaction requested=true 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:31,272 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:31,272 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:31,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:31,273 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:31,273 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:31,273 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,273 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=130.2 K 2024-11-09T17:27:31,273 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,273 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3] 2024-11-09T17:27:31,274 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting b4a9e62cadd54bb9b06363959be4bc0f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244082 2024-11-09T17:27:31,274 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 434e439c9add457b910ab3c407bc3f1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731173244721 2024-11-09T17:27:31,274 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:31,274 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:31,274 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,274 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 52b66510306047698d29be2e5bdbbba8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731173246227 2024-11-09T17:27:31,274 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d9ba6372ae1c42e69554a1f17b85303c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=47.8 K 2024-11-09T17:27:31,275 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 977ce91566514698997b504fd3fceeb3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248417 2024-11-09T17:27:31,275 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d9ba6372ae1c42e69554a1f17b85303c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244082 2024-11-09T17:27:31,275 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9f58db871bdb4718b3b5af372d8e339e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731173244721 2024-11-09T17:27:31,275 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9cd7dd43cedc4af2b52ced4c718d81a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731173246227 2024-11-09T17:27:31,276 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f591f60575504e6cb6747d0005a28171, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248420 2024-11-09T17:27:31,282 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:31,284 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109022e8ce9281241bba5c5f7e9b8a54e3c_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:31,285 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:31,285 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/f1c09aff34dc4affac1d6ff34e0b6beb is 50, key is test_row_0/B:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:31,286 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109022e8ce9281241bba5c5f7e9b8a54e3c_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:31,286 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109022e8ce9281241bba5c5f7e9b8a54e3c_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:31,290 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:31,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742179_1355 (size=12595) 2024-11-09T17:27:31,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=111 2024-11-09T17:27:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,291 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:27:31,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742180_1356 (size=4469) 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:31,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,296 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#296 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:31,296 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5565e19816b34ca6a5abe3de1d7c5a63 is 175, key is test_row_0/A:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110995ce6ded31f64dfaab9fd64332fa49f8_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173249598/Put/seqid=0 2024-11-09T17:27:31,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742181_1357 (size=31549) 2024-11-09T17:27:31,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742182_1358 (size=12304) 2024-11-09T17:27:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:31,342 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110995ce6ded31f64dfaab9fd64332fa49f8_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995ce6ded31f64dfaab9fd64332fa49f8_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:31,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/06c5564392f740958b9b1d2a2b12ec09, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:31,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/06c5564392f740958b9b1d2a2b12ec09 is 175, key is test_row_0/A:col10/1731173249598/Put/seqid=0 2024-11-09T17:27:31,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742183_1359 (size=31105) 2024-11-09T17:27:31,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/06c5564392f740958b9b1d2a2b12ec09 2024-11-09T17:27:31,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6c66ab71a1144c3cb41723e7a4b9fbc1 is 50, key is test_row_0/B:col10/1731173249598/Put/seqid=0 2024-11-09T17:27:31,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742184_1360 (size=12151) 2024-11-09T17:27:31,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:31,695 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/f1c09aff34dc4affac1d6ff34e0b6beb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f1c09aff34dc4affac1d6ff34e0b6beb 2024-11-09T17:27:31,699 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into f1c09aff34dc4affac1d6ff34e0b6beb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:31,699 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:31,699 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=12, startTime=1731173251272; duration=0sec 2024-11-09T17:27:31,699 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:31,699 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:31,700 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:31,701 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:31,701 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:31,701 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,701 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/36e95ac3377f4ed694718822c1b9fd5c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=47.8 K 2024-11-09T17:27:31,701 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 36e95ac3377f4ed694718822c1b9fd5c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731173244082 2024-11-09T17:27:31,701 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0962eb067abc4629b0adefe4515a2fe9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731173244721 2024-11-09T17:27:31,701 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2637d48a98854ab698e1985ea8ed1d31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731173246227 2024-11-09T17:27:31,702 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 88bdf5958b2e4f629b9a96b78820ab5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248420 2024-11-09T17:27:31,717 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:31,717 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5565e19816b34ca6a5abe3de1d7c5a63 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63 2024-11-09T17:27:31,717 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0fd6f0db2a024d67910da843a0d283dd is 50, key is test_row_0/C:col10/1731173248420/Put/seqid=0 2024-11-09T17:27:31,724 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into 5565e19816b34ca6a5abe3de1d7c5a63(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:31,724 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:31,724 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=12, startTime=1731173251272; duration=0sec 2024-11-09T17:27:31,724 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:31,724 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:31,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742185_1361 (size=12595) 2024-11-09T17:27:31,733 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0fd6f0db2a024d67910da843a0d283dd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0fd6f0db2a024d67910da843a0d283dd 2024-11-09T17:27:31,738 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into 0fd6f0db2a024d67910da843a0d283dd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:31,738 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:31,738 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=12, startTime=1731173251272; duration=0sec 2024-11-09T17:27:31,739 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:31,739 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:31,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:31,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6c66ab71a1144c3cb41723e7a4b9fbc1 2024-11-09T17:27:31,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/df9c6eaad347492890ec573657ea7521 is 50, key is test_row_0/C:col10/1731173249598/Put/seqid=0 2024-11-09T17:27:31,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742186_1362 (size=12151) 2024-11-09T17:27:31,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/df9c6eaad347492890ec573657ea7521 2024-11-09T17:27:31,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/06c5564392f740958b9b1d2a2b12ec09 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09 2024-11-09T17:27:31,806 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09, entries=150, sequenceid=211, filesize=30.4 K 2024-11-09T17:27:31,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6c66ab71a1144c3cb41723e7a4b9fbc1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1 2024-11-09T17:27:31,811 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T17:27:31,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/df9c6eaad347492890ec573657ea7521 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521 2024-11-09T17:27:31,818 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T17:27:31,818 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=87.22 KB/89310 for b5327c23ff0d8e9e58fbdf61393b5182 in 528ms, sequenceid=211, compaction requested=false 2024-11-09T17:27:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:31,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:31,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=111}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=111 2024-11-09T17:27:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=111 2024-11-09T17:27:31,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=111, resume processing ppid=110 2024-11-09T17:27:31,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=111, ppid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4420 sec 2024-11-09T17:27:31,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=110, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=110, table=TestAcidGuarantees in 1.4460 sec 2024-11-09T17:27:31,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:31,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:31,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:31,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c6f306a94bba48f79d97faa5c2eca520_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:31,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742187_1363 (size=14794) 2024-11-09T17:27:31,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:31,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173311914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173312024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173312227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:32,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c6f306a94bba48f79d97faa5c2eca520_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c6f306a94bba48f79d97faa5c2eca520_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:32,242 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/0fd386b5597f435d893697492d2a84cc, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:32,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/0fd386b5597f435d893697492d2a84cc is 175, key is test_row_0/A:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:32,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742188_1364 (size=39749) 2024-11-09T17:27:32,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-09T17:27:32,480 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 110 completed 2024-11-09T17:27:32,481 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:32,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees 2024-11-09T17:27:32,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=112, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:32,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:32,482 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=112, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:32,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:32,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173312532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:32,633 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,646 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/0fd386b5597f435d893697492d2a84cc 2024-11-09T17:27:32,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6ffb1eacebb3471595dcc1cb1c59254d is 50, key is test_row_0/B:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742189_1365 (size=12151) 2024-11-09T17:27:32,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:32,786 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:32,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:32,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50056 deadline: 1731173312881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,887 DEBUG [Thread-1397 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8160 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50044 deadline: 1731173312887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,896 DEBUG [Thread-1399 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8174 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:32,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50020 deadline: 1731173312902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,906 DEBUG [Thread-1395 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8184 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:32,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50072 deadline: 1731173312910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,916 DEBUG [Thread-1393 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8191 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:32,937 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:32,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:32,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:32,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173313037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:33,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6ffb1eacebb3471595dcc1cb1c59254d 2024-11-09T17:27:33,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/21634c3ac49f4791bc5b0b25dc98c74e is 50, key is test_row_0/C:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:33,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742190_1366 (size=12151) 2024-11-09T17:27:33,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:33,090 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:33,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,242 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:33,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:33,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,394 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:33,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:33,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:33,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] handler.RSProcedureHandler(58): pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=113 java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=113 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:33,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/21634c3ac49f4791bc5b0b25dc98c74e 2024-11-09T17:27:33,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/0fd386b5597f435d893697492d2a84cc as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc 2024-11-09T17:27:33,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc, entries=200, sequenceid=231, filesize=38.8 K 2024-11-09T17:27:33,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/6ffb1eacebb3471595dcc1cb1c59254d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d 2024-11-09T17:27:33,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T17:27:33,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/21634c3ac49f4791bc5b0b25dc98c74e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e 2024-11-09T17:27:33,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T17:27:33,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b5327c23ff0d8e9e58fbdf61393b5182 in 1654ms, sequenceid=231, compaction requested=true 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:33,478 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:33,478 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:33,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:33,478 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:33,478 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:33,479 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,479 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,479 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=100.0 K 2024-11-09T17:27:33,479 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f1c09aff34dc4affac1d6ff34e0b6beb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=36.0 K 2024-11-09T17:27:33,479 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc] 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5565e19816b34ca6a5abe3de1d7c5a63, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248420 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f1c09aff34dc4affac1d6ff34e0b6beb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248420 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 6c66ab71a1144c3cb41723e7a4b9fbc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173249598 2024-11-09T17:27:33,479 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 06c5564392f740958b9b1d2a2b12ec09, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173249598 2024-11-09T17:27:33,480 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0fd386b5597f435d893697492d2a84cc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:33,480 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 6ffb1eacebb3471595dcc1cb1c59254d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:33,486 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:33,489 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:33,489 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/1555f3b8db08413c8cbb6d7d8404991e is 50, key is test_row_0/B:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:33,490 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109880d622b8ea04774ac81225d9d58b9b6_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:33,492 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109880d622b8ea04774ac81225d9d58b9b6_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:33,492 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109880d622b8ea04774ac81225d9d58b9b6_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:33,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742191_1367 (size=12697) 2024-11-09T17:27:33,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742192_1368 (size=4469) 2024-11-09T17:27:33,497 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#305 average throughput is 2.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:33,498 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7a75290dbf264a0d918ace90f0b26999 is 175, key is test_row_0/A:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:33,500 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/1555f3b8db08413c8cbb6d7d8404991e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/1555f3b8db08413c8cbb6d7d8404991e 2024-11-09T17:27:33,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742193_1369 (size=31651) 2024-11-09T17:27:33,503 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into 1555f3b8db08413c8cbb6d7d8404991e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:33,503 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:33,503 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=13, startTime=1731173253478; duration=0sec 2024-11-09T17:27:33,504 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:33,504 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:33,504 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:33,504 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:33,504 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:33,505 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,505 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0fd6f0db2a024d67910da843a0d283dd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=36.0 K 2024-11-09T17:27:33,505 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0fd6f0db2a024d67910da843a0d283dd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731173248420 2024-11-09T17:27:33,505 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting df9c6eaad347492890ec573657ea7521, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173249598 2024-11-09T17:27:33,505 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 21634c3ac49f4791bc5b0b25dc98c74e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:33,511 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#307 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:33,511 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0279aa8c79404f91bc903ec6fa8539b3 is 50, key is test_row_0/C:col10/1731173251788/Put/seqid=0 2024-11-09T17:27:33,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742194_1370 (size=12697) 2024-11-09T17:27:33,547 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:33,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=113 2024-11-09T17:27:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:33,547 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T17:27:33,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:33,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:33,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fc97a440694d4696becfc2f158d265cf_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173251904/Put/seqid=0 2024-11-09T17:27:33,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742195_1371 (size=12304) 2024-11-09T17:27:33,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:33,906 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/7a75290dbf264a0d918ace90f0b26999 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999 2024-11-09T17:27:33,910 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into 7a75290dbf264a0d918ace90f0b26999(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:33,910 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:33,910 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=13, startTime=1731173253478; duration=0sec 2024-11-09T17:27:33,910 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:33,910 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:33,918 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0279aa8c79404f91bc903ec6fa8539b3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0279aa8c79404f91bc903ec6fa8539b3 2024-11-09T17:27:33,922 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into 0279aa8c79404f91bc903ec6fa8539b3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:33,922 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:33,922 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=13, startTime=1731173253478; duration=0sec 2024-11-09T17:27:33,922 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:33,922 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:33,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:33,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fc97a440694d4696becfc2f158d265cf_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc97a440694d4696becfc2f158d265cf_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:33,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/d7ccac42805642da9172c1d2df3e2a7d, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/d7ccac42805642da9172c1d2df3e2a7d is 175, key is test_row_0/A:col10/1731173251904/Put/seqid=0 2024-11-09T17:27:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742196_1372 (size=31105) 2024-11-09T17:27:33,965 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/d7ccac42805642da9172c1d2df3e2a7d 2024-11-09T17:27:33,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a4a3c3998ed448dd92d0ffa5cfa03f6f is 50, key is test_row_0/B:col10/1731173251904/Put/seqid=0 2024-11-09T17:27:33,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742197_1373 (size=12151) 2024-11-09T17:27:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:34,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. as already flushing 2024-11-09T17:27:34,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173314101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173314206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:34,378 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a4a3c3998ed448dd92d0ffa5cfa03f6f 2024-11-09T17:27:34,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0238a4dea61e4611a4781e368073ee14 is 50, key is test_row_0/C:col10/1731173251904/Put/seqid=0 2024-11-09T17:27:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742198_1374 (size=12151) 2024-11-09T17:27:34,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:34,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173314410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:34,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173314713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:34,788 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0238a4dea61e4611a4781e368073ee14 2024-11-09T17:27:34,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/d7ccac42805642da9172c1d2df3e2a7d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d 2024-11-09T17:27:34,795 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d, entries=150, sequenceid=251, filesize=30.4 K 2024-11-09T17:27:34,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/a4a3c3998ed448dd92d0ffa5cfa03f6f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f 2024-11-09T17:27:34,799 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T17:27:34,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/0238a4dea61e4611a4781e368073ee14 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14 2024-11-09T17:27:34,810 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14, entries=150, sequenceid=251, filesize=11.9 K 2024-11-09T17:27:34,811 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b5327c23ff0d8e9e58fbdf61393b5182 in 1264ms, sequenceid=251, compaction requested=false 2024-11-09T17:27:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:34,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-11-09T17:27:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=113 2024-11-09T17:27:34,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=113, resume processing ppid=112 2024-11-09T17:27:34,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=113, ppid=112, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3310 sec 2024-11-09T17:27:34,819 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=112, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=112, table=TestAcidGuarantees in 2.3330 sec 2024-11-09T17:27:35,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:35,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:35,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:35,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110967591d84ad77435ab83fdf1ebfb65370_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:35,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742199_1375 (size=14994) 2024-11-09T17:27:35,233 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:35,236 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110967591d84ad77435ab83fdf1ebfb65370_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967591d84ad77435ab83fdf1ebfb65370_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:35,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/99179df7a21d41038a46e043bf9cf25f, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:35,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/99179df7a21d41038a46e043bf9cf25f is 175, key is test_row_0/A:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742200_1376 (size=39949) 2024-11-09T17:27:35,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:35,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173315287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:35,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173315392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173315597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:35,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/99179df7a21d41038a46e043bf9cf25f 2024-11-09T17:27:35,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/10d8c52747da4197a2c772b04e4b1174 is 50, key is test_row_0/B:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742201_1377 (size=12301) 2024-11-09T17:27:35,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:35,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173315903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:35,951 DEBUG [Thread-1404 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f49665c to 127.0.0.1:57222 2024-11-09T17:27:35,951 DEBUG [Thread-1404 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:35,955 DEBUG [Thread-1412 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68035c67 to 127.0.0.1:57222 2024-11-09T17:27:35,955 DEBUG [Thread-1412 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:35,956 DEBUG [Thread-1408 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75e4d3d0 to 127.0.0.1:57222 2024-11-09T17:27:35,956 DEBUG [Thread-1408 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:35,956 DEBUG [Thread-1410 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2b308f62 to 127.0.0.1:57222 2024-11-09T17:27:35,956 DEBUG [Thread-1410 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:35,960 DEBUG [Thread-1406 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x683f8469 to 127.0.0.1:57222 2024-11-09T17:27:35,960 DEBUG [Thread-1406 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:36,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/10d8c52747da4197a2c772b04e4b1174 2024-11-09T17:27:36,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/ae243ef634b84878a6b9b936353fe88d is 50, key is test_row_0/C:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:36,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742202_1378 (size=12301) 2024-11-09T17:27:36,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50088 deadline: 1731173316411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:36,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/ae243ef634b84878a6b9b936353fe88d 2024-11-09T17:27:36,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/99179df7a21d41038a46e043bf9cf25f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f 2024-11-09T17:27:36,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f, entries=200, sequenceid=271, filesize=39.0 K 2024-11-09T17:27:36,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/10d8c52747da4197a2c772b04e4b1174 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174 2024-11-09T17:27:36,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174, entries=150, sequenceid=271, filesize=12.0 K 2024-11-09T17:27:36,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/ae243ef634b84878a6b9b936353fe88d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d 2024-11-09T17:27:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d, entries=150, sequenceid=271, filesize=12.0 K 2024-11-09T17:27:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b5327c23ff0d8e9e58fbdf61393b5182 in 1294ms, sequenceid=271, compaction requested=true 2024-11-09T17:27:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b5327c23ff0d8e9e58fbdf61393b5182:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:36,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/B is initiating minor compaction (all files) 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:36,515 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/B in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:36,515 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/A is initiating minor compaction (all files) 2024-11-09T17:27:36,515 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/A in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:36,516 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/1555f3b8db08413c8cbb6d7d8404991e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=36.3 K 2024-11-09T17:27:36,516 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=100.3 K 2024-11-09T17:27:36,516 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f] 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1555f3b8db08413c8cbb6d7d8404991e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 7a75290dbf264a0d918ace90f0b26999, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a4a3c3998ed448dd92d0ffa5cfa03f6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731173251898 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d7ccac42805642da9172c1d2df3e2a7d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731173251898 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 10d8c52747da4197a2c772b04e4b1174, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731173254083 2024-11-09T17:27:36,516 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 99179df7a21d41038a46e043bf9cf25f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731173254083 2024-11-09T17:27:36,521 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:36,523 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:36,523 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/98d30773fdc047649edd03143997e95e is 50, key is test_row_0/B:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:36,524 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110989cced9eceb54f03b2fca852cf015ab1_b5327c23ff0d8e9e58fbdf61393b5182 store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:36,527 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110989cced9eceb54f03b2fca852cf015ab1_b5327c23ff0d8e9e58fbdf61393b5182, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:36,528 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110989cced9eceb54f03b2fca852cf015ab1_b5327c23ff0d8e9e58fbdf61393b5182 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:36,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742203_1379 (size=12949) 2024-11-09T17:27:36,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742204_1380 (size=4469) 2024-11-09T17:27:36,582 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:27:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-09T17:27:36,587 INFO [Thread-1403 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 112 completed 2024-11-09T17:27:36,934 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#A#compaction#314 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:36,936 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/003bb69a26f94ecfb532e9b3d85cb6bb is 175, key is test_row_0/A:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:36,940 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/98d30773fdc047649edd03143997e95e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/98d30773fdc047649edd03143997e95e 2024-11-09T17:27:36,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742205_1381 (size=31903) 2024-11-09T17:27:36,945 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/B of b5327c23ff0d8e9e58fbdf61393b5182 into 98d30773fdc047649edd03143997e95e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:36,945 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:36,945 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/B, priority=13, startTime=1731173256515; duration=0sec 2024-11-09T17:27:36,945 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:36,945 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:B 2024-11-09T17:27:36,945 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:36,946 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:36,946 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): b5327c23ff0d8e9e58fbdf61393b5182/C is initiating minor compaction (all files) 2024-11-09T17:27:36,946 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of b5327c23ff0d8e9e58fbdf61393b5182/C in TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:36,946 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0279aa8c79404f91bc903ec6fa8539b3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp, totalSize=36.3 K 2024-11-09T17:27:36,946 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0279aa8c79404f91bc903ec6fa8539b3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173251788 2024-11-09T17:27:36,947 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 0238a4dea61e4611a4781e368073ee14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731173251898 2024-11-09T17:27:36,947 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ae243ef634b84878a6b9b936353fe88d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731173254083 2024-11-09T17:27:36,953 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b5327c23ff0d8e9e58fbdf61393b5182#C#compaction#316 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:36,953 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/e1e70ada898a4dda8af510caa9733d93 is 50, key is test_row_0/C:col10/1731173254083/Put/seqid=0 2024-11-09T17:27:36,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742206_1382 (size=12949) 2024-11-09T17:27:37,353 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/003bb69a26f94ecfb532e9b3d85cb6bb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/003bb69a26f94ecfb532e9b3d85cb6bb 2024-11-09T17:27:37,361 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/A of b5327c23ff0d8e9e58fbdf61393b5182 into 003bb69a26f94ecfb532e9b3d85cb6bb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:37,361 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:37,361 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/A, priority=13, startTime=1731173256514; duration=0sec 2024-11-09T17:27:37,361 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:37,361 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:A 2024-11-09T17:27:37,363 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/e1e70ada898a4dda8af510caa9733d93 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/e1e70ada898a4dda8af510caa9733d93 2024-11-09T17:27:37,368 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in b5327c23ff0d8e9e58fbdf61393b5182/C of b5327c23ff0d8e9e58fbdf61393b5182 into e1e70ada898a4dda8af510caa9733d93(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:37,368 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:37,368 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182., storeName=b5327c23ff0d8e9e58fbdf61393b5182/C, priority=13, startTime=1731173256515; duration=0sec 2024-11-09T17:27:37,368 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:37,369 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b5327c23ff0d8e9e58fbdf61393b5182:C 2024-11-09T17:27:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:37,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-09T17:27:37,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:37,423 DEBUG [Thread-1401 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f94d721 to 127.0.0.1:57222 2024-11-09T17:27:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:37,424 DEBUG [Thread-1401 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:37,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:37,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110968a98b7ba07c4766aa8244519b58d5fa_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173257421/Put/seqid=0 2024-11-09T17:27:37,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742207_1383 (size=12454) 2024-11-09T17:27:37,839 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:37,847 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110968a98b7ba07c4766aa8244519b58d5fa_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968a98b7ba07c4766aa8244519b58d5fa_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:37,849 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3c01d3787c1b46988b80dbefa31eef09, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:37,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3c01d3787c1b46988b80dbefa31eef09 is 175, key is test_row_0/A:col10/1731173257421/Put/seqid=0 2024-11-09T17:27:37,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742208_1384 (size=31255) 2024-11-09T17:27:38,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3c01d3787c1b46988b80dbefa31eef09 2024-11-09T17:27:38,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/097252152e794e47856c79090c8215a8 is 50, key is test_row_0/B:col10/1731173257421/Put/seqid=0 2024-11-09T17:27:38,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742209_1385 (size=12301) 2024-11-09T17:27:38,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/097252152e794e47856c79090c8215a8 2024-11-09T17:27:38,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/45d19265171c411d9df4728994a412ad is 50, key is test_row_0/C:col10/1731173257421/Put/seqid=0 2024-11-09T17:27:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742210_1386 (size=12301) 2024-11-09T17:27:39,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/45d19265171c411d9df4728994a412ad 2024-11-09T17:27:39,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/3c01d3787c1b46988b80dbefa31eef09 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3c01d3787c1b46988b80dbefa31eef09 2024-11-09T17:27:39,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3c01d3787c1b46988b80dbefa31eef09, entries=150, sequenceid=293, filesize=30.5 K 2024-11-09T17:27:39,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/097252152e794e47856c79090c8215a8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/097252152e794e47856c79090c8215a8 2024-11-09T17:27:39,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/097252152e794e47856c79090c8215a8, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T17:27:39,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/45d19265171c411d9df4728994a412ad as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/45d19265171c411d9df4728994a412ad 2024-11-09T17:27:39,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/45d19265171c411d9df4728994a412ad, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T17:27:39,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for b5327c23ff0d8e9e58fbdf61393b5182 in 1691ms, sequenceid=293, compaction requested=false 2024-11-09T17:27:39,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:40,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f] to archive 2024-11-09T17:27:40,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:40,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/aba4dd79257f43b98be0b8afb8d9d61f 2024-11-09T17:27:40,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/231ff27dff594d3395eda798325e427d 2024-11-09T17:27:40,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7fe52d55113b4961bf0138554a928a94 2024-11-09T17:27:40,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b1d88e49666948c19410354cd79c8006 2024-11-09T17:27:40,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/cc03eac6f9f84eb396f96c979b4a7eb9 2024-11-09T17:27:40,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/e6ddfd1a1b06407aab1c7639afcf9c05 2024-11-09T17:27:40,963 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3f95d9b19de141c9a88fc972e5402dc4 2024-11-09T17:27:40,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/2c497b5a7cc34d2ebb9629d03c92cfc7 2024-11-09T17:27:40,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/9141ae54f50d47e082aedaf745c4b66f 2024-11-09T17:27:40,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/b4a9e62cadd54bb9b06363959be4bc0f 2024-11-09T17:27:40,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/434e439c9add457b910ab3c407bc3f1e 2024-11-09T17:27:40,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/52b66510306047698d29be2e5bdbbba8 2024-11-09T17:27:40,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/977ce91566514698997b504fd3fceeb3 2024-11-09T17:27:40,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5565e19816b34ca6a5abe3de1d7c5a63 2024-11-09T17:27:40,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/06c5564392f740958b9b1d2a2b12ec09 2024-11-09T17:27:40,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/0fd386b5597f435d893697492d2a84cc 2024-11-09T17:27:40,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/7a75290dbf264a0d918ace90f0b26999 2024-11-09T17:27:40,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/d7ccac42805642da9172c1d2df3e2a7d 2024-11-09T17:27:40,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/99179df7a21d41038a46e043bf9cf25f 2024-11-09T17:27:40,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/c04512ca14494d21800c3620b26e20cd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cc7cea3309f4d2598ba220e9f4c2ba3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d9ba6372ae1c42e69554a1f17b85303c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f1c09aff34dc4affac1d6ff34e0b6beb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/1555f3b8db08413c8cbb6d7d8404991e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174] to archive 2024-11-09T17:27:40,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:40,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2d72cd651f27440ab2c142f960367692 2024-11-09T17:27:40,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a127d47a2e8648c4b25f08c553c8565d 2024-11-09T17:27:40,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/c04512ca14494d21800c3620b26e20cd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/c04512ca14494d21800c3620b26e20cd 2024-11-09T17:27:40,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d21559e5c7f043508c27fd82b890a58e 2024-11-09T17:27:40,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/40d6920439b24f0fb871a1a70d9c8007 2024-11-09T17:27:40,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cc7cea3309f4d2598ba220e9f4c2ba3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cc7cea3309f4d2598ba220e9f4c2ba3 2024-11-09T17:27:40,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/672fba0735654b6eae228dfc64561680 2024-11-09T17:27:40,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/2db7b2fded644453bb088ba6bdab03b0 2024-11-09T17:27:40,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d9ba6372ae1c42e69554a1f17b85303c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/d9ba6372ae1c42e69554a1f17b85303c 2024-11-09T17:27:40,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/7f0607fea8a54312a642afa9aa452931 2024-11-09T17:27:40,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9f58db871bdb4718b3b5af372d8e339e 2024-11-09T17:27:40,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/9cd7dd43cedc4af2b52ced4c718d81a1 2024-11-09T17:27:40,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f1c09aff34dc4affac1d6ff34e0b6beb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f1c09aff34dc4affac1d6ff34e0b6beb 2024-11-09T17:27:40,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/f591f60575504e6cb6747d0005a28171 2024-11-09T17:27:40,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6c66ab71a1144c3cb41723e7a4b9fbc1 2024-11-09T17:27:40,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/1555f3b8db08413c8cbb6d7d8404991e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/1555f3b8db08413c8cbb6d7d8404991e 2024-11-09T17:27:40,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/6ffb1eacebb3471595dcc1cb1c59254d 2024-11-09T17:27:40,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/a4a3c3998ed448dd92d0ffa5cfa03f6f 2024-11-09T17:27:40,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/10d8c52747da4197a2c772b04e4b1174 2024-11-09T17:27:40,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2e4d1d642b4749838cb11f961b8315db, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ffab9f795c7a4d33a1ee3ba287cd4fd8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/36e95ac3377f4ed694718822c1b9fd5c, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0fd6f0db2a024d67910da843a0d283dd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0279aa8c79404f91bc903ec6fa8539b3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d] to archive 2024-11-09T17:27:40,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:27:40,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/d27b23d4249e42169b12a930142fa758 2024-11-09T17:27:41,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/b076f38b01d3468581da233d74a97cb4 2024-11-09T17:27:41,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2e4d1d642b4749838cb11f961b8315db to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2e4d1d642b4749838cb11f961b8315db 2024-11-09T17:27:41,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/8e14e3958c664396897b34ae5c27dfa9 2024-11-09T17:27:41,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/f3e379021d5145fa8e10a88dc7d4ff30 2024-11-09T17:27:41,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ffab9f795c7a4d33a1ee3ba287cd4fd8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ffab9f795c7a4d33a1ee3ba287cd4fd8 2024-11-09T17:27:41,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0c830365083b4b84a34ad029e69b213f 2024-11-09T17:27:41,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0aa1f0271ea045cab335d56b4f0d51e1 2024-11-09T17:27:41,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/36e95ac3377f4ed694718822c1b9fd5c to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/36e95ac3377f4ed694718822c1b9fd5c 2024-11-09T17:27:41,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/c403f2ab8b8e441a84e81429c4449b3b 2024-11-09T17:27:41,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0962eb067abc4629b0adefe4515a2fe9 2024-11-09T17:27:41,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/2637d48a98854ab698e1985ea8ed1d31 2024-11-09T17:27:41,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0fd6f0db2a024d67910da843a0d283dd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0fd6f0db2a024d67910da843a0d283dd 2024-11-09T17:27:41,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/88bdf5958b2e4f629b9a96b78820ab5a 2024-11-09T17:27:41,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/df9c6eaad347492890ec573657ea7521 2024-11-09T17:27:41,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0279aa8c79404f91bc903ec6fa8539b3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0279aa8c79404f91bc903ec6fa8539b3 2024-11-09T17:27:41,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/21634c3ac49f4791bc5b0b25dc98c74e 2024-11-09T17:27:41,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/0238a4dea61e4611a4781e368073ee14 2024-11-09T17:27:41,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/c28fd47d982f:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/ae243ef634b84878a6b9b936353fe88d 2024-11-09T17:27:42,001 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:42,684 ERROR [LeaseRenewer:jenkins@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46491,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:42,936 DEBUG [Thread-1397 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3dd5b441 to 127.0.0.1:57222 2024-11-09T17:27:42,936 DEBUG [Thread-1397 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:42,959 DEBUG [Thread-1399 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c336ea4 to 127.0.0.1:57222 2024-11-09T17:27:42,959 DEBUG [Thread-1399 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:42,975 DEBUG [Thread-1393 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b6cf8cb to 127.0.0.1:57222 2024-11-09T17:27:42,975 DEBUG [Thread-1393 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:42,992 DEBUG [Thread-1395 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ec15031 to 127.0.0.1:57222 2024-11-09T17:27:42,992 DEBUG [Thread-1395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2162 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6486 rows 2024-11-09T17:27:42,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2167 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6501 rows 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2169 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6507 rows 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2171 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6510 rows 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2166 2024-11-09T17:27:42,994 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 6495 rows 2024-11-09T17:27:42,994 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:27:42,994 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b82ba2a to 127.0.0.1:57222 2024-11-09T17:27:42,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:27:42,997 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:27:42,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:27:42,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:43,002 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173263001"}]},"ts":"1731173263001"} 2024-11-09T17:27:43,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:27:43,019 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:27:43,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:27:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, UNASSIGN}] 2024-11-09T17:27:43,022 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=116, ppid=115, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, UNASSIGN 2024-11-09T17:27:43,023 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=116 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:43,024 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:27:43,024 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; CloseRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:43,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:43,176 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:43,177 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] handler.UnassignRegionHandler(124): Close b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:43,177 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:27:43,177 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1681): Closing b5327c23ff0d8e9e58fbdf61393b5182, disabling compactions & flushes 2024-11-09T17:27:43,177 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:43,178 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:43,178 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. after waiting 0 ms 2024-11-09T17:27:43,178 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:43,178 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(2837): Flushing b5327c23ff0d8e9e58fbdf61393b5182 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T17:27:43,178 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=A 2024-11-09T17:27:43,179 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:43,179 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=B 2024-11-09T17:27:43,179 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:43,179 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b5327c23ff0d8e9e58fbdf61393b5182, store=C 2024-11-09T17:27:43,179 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:43,192 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a3a944606ffd451caebf30516b86486c_b5327c23ff0d8e9e58fbdf61393b5182 is 50, key is test_row_0/A:col10/1731173262958/Put/seqid=0 2024-11-09T17:27:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742211_1387 (size=9914) 2024-11-09T17:27:43,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:43,598 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:43,609 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a3a944606ffd451caebf30516b86486c_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3a944606ffd451caebf30516b86486c_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:43,612 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5901ab8a78d346ec9587158af1dba117, store: [table=TestAcidGuarantees family=A region=b5327c23ff0d8e9e58fbdf61393b5182] 2024-11-09T17:27:43,613 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5901ab8a78d346ec9587158af1dba117 is 175, key is test_row_0/A:col10/1731173262958/Put/seqid=0 2024-11-09T17:27:43,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742212_1388 (size=22561) 2024-11-09T17:27:44,018 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5901ab8a78d346ec9587158af1dba117 2024-11-09T17:27:44,031 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/e24eae2f91424493a4ce2142279cbeb6 is 50, key is test_row_0/B:col10/1731173262958/Put/seqid=0 2024-11-09T17:27:44,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742213_1389 (size=9857) 2024-11-09T17:27:44,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:44,436 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/e24eae2f91424493a4ce2142279cbeb6 2024-11-09T17:27:44,449 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/baa51ade95de44cc884868e94585bed5 is 50, key is test_row_0/C:col10/1731173262958/Put/seqid=0 2024-11-09T17:27:44,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742214_1390 (size=9857) 2024-11-09T17:27:44,853 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/baa51ade95de44cc884868e94585bed5 2024-11-09T17:27:44,863 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/A/5901ab8a78d346ec9587158af1dba117 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5901ab8a78d346ec9587158af1dba117 2024-11-09T17:27:44,867 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5901ab8a78d346ec9587158af1dba117, entries=100, sequenceid=300, filesize=22.0 K 2024-11-09T17:27:44,867 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/B/e24eae2f91424493a4ce2142279cbeb6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/e24eae2f91424493a4ce2142279cbeb6 2024-11-09T17:27:44,871 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/e24eae2f91424493a4ce2142279cbeb6, entries=100, sequenceid=300, filesize=9.6 K 2024-11-09T17:27:44,871 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/.tmp/C/baa51ade95de44cc884868e94585bed5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/baa51ade95de44cc884868e94585bed5 2024-11-09T17:27:44,874 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/baa51ade95de44cc884868e94585bed5, entries=100, sequenceid=300, filesize=9.6 K 2024-11-09T17:27:44,874 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b5327c23ff0d8e9e58fbdf61393b5182 in 1696ms, sequenceid=300, compaction requested=true 2024-11-09T17:27:44,878 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-09T17:27:44,878 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182. 2024-11-09T17:27:44,878 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] regionserver.HRegion(1635): Region close journal for b5327c23ff0d8e9e58fbdf61393b5182: 2024-11-09T17:27:44,879 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=117}] handler.UnassignRegionHandler(170): Closed b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:44,880 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=116 updating hbase:meta row=b5327c23ff0d8e9e58fbdf61393b5182, regionState=CLOSED 2024-11-09T17:27:44,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-11-09T17:27:44,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; CloseRegionProcedure b5327c23ff0d8e9e58fbdf61393b5182, server=c28fd47d982f,45633,1731173138129 in 1.8560 sec 2024-11-09T17:27:44,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T17:27:44,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, ppid=115, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=b5327c23ff0d8e9e58fbdf61393b5182, UNASSIGN in 1.8600 sec 2024-11-09T17:27:44,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-11-09T17:27:44,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-09T17:27:44,885 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173264885"}]},"ts":"1731173264885"} 2024-11-09T17:27:44,885 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:27:44,928 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:27:44,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9300 sec 2024-11-09T17:27:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-09T17:27:45,109 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 114 completed 2024-11-09T17:27:45,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:27:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,111 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=118, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-09T17:27:45,112 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=118, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,114 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,117 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits] 2024-11-09T17:27:45,119 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/003bb69a26f94ecfb532e9b3d85cb6bb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/003bb69a26f94ecfb532e9b3d85cb6bb 2024-11-09T17:27:45,121 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3c01d3787c1b46988b80dbefa31eef09 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/3c01d3787c1b46988b80dbefa31eef09 2024-11-09T17:27:45,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5901ab8a78d346ec9587158af1dba117 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/A/5901ab8a78d346ec9587158af1dba117 2024-11-09T17:27:45,124 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/097252152e794e47856c79090c8215a8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/097252152e794e47856c79090c8215a8 2024-11-09T17:27:45,125 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/98d30773fdc047649edd03143997e95e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/98d30773fdc047649edd03143997e95e 2024-11-09T17:27:45,126 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/e24eae2f91424493a4ce2142279cbeb6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/B/e24eae2f91424493a4ce2142279cbeb6 2024-11-09T17:27:45,129 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/45d19265171c411d9df4728994a412ad to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/45d19265171c411d9df4728994a412ad 2024-11-09T17:27:45,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/baa51ade95de44cc884868e94585bed5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/baa51ade95de44cc884868e94585bed5 2024-11-09T17:27:45,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/e1e70ada898a4dda8af510caa9733d93 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/C/e1e70ada898a4dda8af510caa9733d93 2024-11-09T17:27:45,135 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits/303.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182/recovered.edits/303.seqid 2024-11-09T17:27:45,136 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,136 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:27:45,136 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:27:45,137 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T17:27:45,141 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b301dd617474eecacc73b8fa4fc16cd_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b301dd617474eecacc73b8fa4fc16cd_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,143 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938673eca34a3495898b3a91e158e3dd2_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110938673eca34a3495898b3a91e158e3dd2_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,145 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e82141f585c4ff8ae4c102672ea1730_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093e82141f585c4ff8ae4c102672ea1730_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,147 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc1d73696804157a2ed44b9b6ba9697_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094fc1d73696804157a2ed44b9b6ba9697_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,148 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967591d84ad77435ab83fdf1ebfb65370_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110967591d84ad77435ab83fdf1ebfb65370_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,150 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968a98b7ba07c4766aa8244519b58d5fa_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110968a98b7ba07c4766aa8244519b58d5fa_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096c7645ca350e47528ecd31db9526c251_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411096c7645ca350e47528ecd31db9526c251_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,154 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097126030293d54bca970c18e8d8887e6a_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097126030293d54bca970c18e8d8887e6a_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c77f7fcda49432ba40958a1d54a07a2_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c77f7fcda49432ba40958a1d54a07a2_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989ea38f76fcd46ccb1eceb7ea2d04473_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989ea38f76fcd46ccb1eceb7ea2d04473_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110993faa00fc34543b7b9fd773a0acbce41_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110993faa00fc34543b7b9fd773a0acbce41_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995ce6ded31f64dfaab9fd64332fa49f8_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110995ce6ded31f64dfaab9fd64332fa49f8_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fd184d0b0bf4120b428dd2f6634fd55_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099fd184d0b0bf4120b428dd2f6634fd55_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3a944606ffd451caebf30516b86486c_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a3a944606ffd451caebf30516b86486c_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c6f306a94bba48f79d97faa5c2eca520_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c6f306a94bba48f79d97faa5c2eca520_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc97a440694d4696becfc2f158d265cf_b5327c23ff0d8e9e58fbdf61393b5182 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc97a440694d4696becfc2f158d265cf_b5327c23ff0d8e9e58fbdf61393b5182 2024-11-09T17:27:45,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:27:45,165 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=118, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,166 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:27:45,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:27:45,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=118, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:27:45,169 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173265168"}]},"ts":"9223372036854775807"} 2024-11-09T17:27:45,170 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:27:45,170 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => b5327c23ff0d8e9e58fbdf61393b5182, NAME => 'TestAcidGuarantees,,1731173232675.b5327c23ff0d8e9e58fbdf61393b5182.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:27:45,170 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:27:45,170 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173265170"}]},"ts":"9223372036854775807"} 2024-11-09T17:27:45,171 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:27:45,178 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=118, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=118, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-09T17:27:45,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-09T17:27:45,213 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 118 completed 2024-11-09T17:27:45,224 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobScanAtomicity Thread=236 (was 238), OpenFileDescriptor=446 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=303 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=5835 (was 5892) 2024-11-09T17:27:45,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testScanAtomicity Thread=236, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=5834 2024-11-09T17:27:45,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:27:45,237 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:27:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:27:45,239 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:27:45,239 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:45,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 119 2024-11-09T17:27:45,240 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:27:45,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:45,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742215_1391 (size=960) 2024-11-09T17:27:45,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:45,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:45,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:27:45,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742216_1392 (size=53) 2024-11-09T17:27:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:46,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:27:46,064 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing d57ffb67cf090d609649a6e4d95584ca, disabling compactions & flushes 2024-11-09T17:27:46,064 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. after waiting 0 ms 2024-11-09T17:27:46,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,065 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,065 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:46,067 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:27:46,067 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173266067"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173266067"}]},"ts":"1731173266067"} 2024-11-09T17:27:46,069 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:27:46,070 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:27:46,071 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173266070"}]},"ts":"1731173266070"} 2024-11-09T17:27:46,072 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:27:46,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, ASSIGN}] 2024-11-09T17:27:46,131 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=120, ppid=119, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, ASSIGN 2024-11-09T17:27:46,132 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=120, ppid=119, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:27:46,283 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=120 updating hbase:meta row=d57ffb67cf090d609649a6e4d95584ca, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:46,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE; OpenRegionProcedure d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:27:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:46,441 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:46,447 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,448 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(7285): Opening region: {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:27:46,448 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,449 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:27:46,449 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(7327): checking encryption for d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,449 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(7330): checking classloading for d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,451 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,453 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:46,453 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57ffb67cf090d609649a6e4d95584ca columnFamilyName A 2024-11-09T17:27:46,453 DEBUG [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:46,454 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(327): Store=d57ffb67cf090d609649a6e4d95584ca/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:46,454 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,456 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:46,456 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57ffb67cf090d609649a6e4d95584ca columnFamilyName B 2024-11-09T17:27:46,456 DEBUG [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:46,457 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(327): Store=d57ffb67cf090d609649a6e4d95584ca/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:46,457 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,458 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:27:46,459 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d57ffb67cf090d609649a6e4d95584ca columnFamilyName C 2024-11-09T17:27:46,459 DEBUG [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:27:46,459 INFO [StoreOpener-d57ffb67cf090d609649a6e4d95584ca-1 {}] regionserver.HStore(327): Store=d57ffb67cf090d609649a6e4d95584ca/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:27:46,460 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,461 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,461 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,463 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:27:46,464 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(1085): writing seq id for d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:46,465 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:27:46,465 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(1102): Opened d57ffb67cf090d609649a6e4d95584ca; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62833676, jitterRate=-0.06370526552200317}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:27:46,466 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegion(1001): Region open journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:46,466 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., pid=121, masterSystemTime=1731173266441 2024-11-09T17:27:46,467 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,467 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=121}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:46,468 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=120 updating hbase:meta row=d57ffb67cf090d609649a6e4d95584ca, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:46,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=121, resume processing ppid=120 2024-11-09T17:27:46,469 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, ppid=120, state=SUCCESS; OpenRegionProcedure d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 in 182 msec 2024-11-09T17:27:46,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T17:27:46,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, ASSIGN in 341 msec 2024-11-09T17:27:46,471 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:27:46,471 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173266471"}]},"ts":"1731173266471"} 2024-11-09T17:27:46,471 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:27:46,546 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=119, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:27:46,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.3090 sec 2024-11-09T17:27:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-09T17:27:47,351 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 119 completed 2024-11-09T17:27:47,354 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x63cefe40 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32c12a30 2024-11-09T17:27:47,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b10416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,410 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36594, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:27:47,414 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50024, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:27:47,416 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f142b04 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61d38088 2024-11-09T17:27:47,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0ab200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,440 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0de9f076 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7043f683 2024-11-09T17:27:47,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5871c039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,455 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4414259d to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b0c2472 2024-11-09T17:27:47,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7daa5922, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,463 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ed69825 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34b30c39 2024-11-09T17:27:47,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b7f20c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,472 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11193a0c to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d672ed2 2024-11-09T17:27:47,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f7c40ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,480 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7cf40102 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@41b0e7b6 2024-11-09T17:27:47,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6050584c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,490 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x496fe03f to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f2423f3 2024-11-09T17:27:47,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd48863, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,504 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3652e74d to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@184771cf 2024-11-09T17:27:47,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51196534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,512 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2405c04e to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@76f0408 2024-11-09T17:27:47,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dc5e114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73d92042 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c692575 2024-11-09T17:27:47,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e96b8ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:27:47,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:47,531 DEBUG [hconnection-0x1738bae5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,531 DEBUG [hconnection-0x14226508-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,532 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36602, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees 2024-11-09T17:27:47,532 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36608, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:47,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:47,533 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=122, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:47,533 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:47,534 DEBUG [hconnection-0x13c37e18-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,534 DEBUG [hconnection-0x4ab49c6d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,534 DEBUG [hconnection-0x4b946a48-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,535 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36610, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,535 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36612, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,535 DEBUG [hconnection-0xea7d11d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,535 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36642, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,535 DEBUG [hconnection-0x32fd34df-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,536 DEBUG [hconnection-0x21291153-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,536 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36652, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,536 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36662, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,536 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36668, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:47,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:27:47,539 DEBUG [hconnection-0x51c7d23a-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:47,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:47,539 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36682, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,543 DEBUG [hconnection-0x4380b8d-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:27:47,544 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36686, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:27:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173327546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173327547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173327547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173327547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173327547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/f8823d2d6fc748cabdd17d02a7bb0ede is 50, key is test_row_0/A:col10/1731173267536/Put/seqid=0 2024-11-09T17:27:47,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742217_1393 (size=12001) 2024-11-09T17:27:47,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/f8823d2d6fc748cabdd17d02a7bb0ede 2024-11-09T17:27:47,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/cf9fdbb3148f4fb99798287cc6800041 is 50, key is test_row_0/B:col10/1731173267536/Put/seqid=0 2024-11-09T17:27:47,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742218_1394 (size=12001) 2024-11-09T17:27:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173327648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173327648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173327648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173327648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173327648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,686 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:47,838 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173327851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173327851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173327852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173327851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:47,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173327853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,991 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:47,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:47,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:47,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:47,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:47,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/cf9fdbb3148f4fb99798287cc6800041 2024-11-09T17:27:48,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/dabdcccba2b84a76abee03417195ee2d is 50, key is test_row_0/C:col10/1731173267536/Put/seqid=0 2024-11-09T17:27:48,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742219_1395 (size=12001) 2024-11-09T17:27:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:48,143 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:48,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:48,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:48,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:48,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173328153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173328153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173328156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173328157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173328158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,295 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:48,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] handler.RSProcedureHandler(58): pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=123 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=123 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:48,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/dabdcccba2b84a76abee03417195ee2d 2024-11-09T17:27:48,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/f8823d2d6fc748cabdd17d02a7bb0ede as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede 2024-11-09T17:27:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T17:27:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/cf9fdbb3148f4fb99798287cc6800041 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041 2024-11-09T17:27:48,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T17:27:48,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/dabdcccba2b84a76abee03417195ee2d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d 2024-11-09T17:27:48,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T17:27:48,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d57ffb67cf090d609649a6e4d95584ca in 905ms, sequenceid=13, compaction requested=false 2024-11-09T17:27:48,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:48,448 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=123 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:48,448 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:48,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:48,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:48,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/49554d7dc33d4957a882009c5e8118e6 is 50, key is test_row_0/A:col10/1731173267546/Put/seqid=0 2024-11-09T17:27:48,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742220_1396 (size=12001) 2024-11-09T17:27:48,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:48,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:48,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173328666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173328666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173328667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173328670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173328670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173328771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173328771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173328772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173328773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,855 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/49554d7dc33d4957a882009c5e8118e6 2024-11-09T17:27:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/58b105c1677f448496155e6fd13f89b0 is 50, key is test_row_0/B:col10/1731173267546/Put/seqid=0 2024-11-09T17:27:48,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742221_1397 (size=12001) 2024-11-09T17:27:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173328974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173328974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173328977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173328978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,264 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/58b105c1677f448496155e6fd13f89b0 2024-11-09T17:27:49,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a27fde54afd8498d94a8d247e6737cfe is 50, key is test_row_0/C:col10/1731173267546/Put/seqid=0 2024-11-09T17:27:49,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742222_1398 (size=12001) 2024-11-09T17:27:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173329280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173329281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173329281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173329282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:49,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a27fde54afd8498d94a8d247e6737cfe 2024-11-09T17:27:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173329672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/49554d7dc33d4957a882009c5e8118e6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6 2024-11-09T17:27:49,680 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T17:27:49,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/58b105c1677f448496155e6fd13f89b0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0 2024-11-09T17:27:49,683 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T17:27:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a27fde54afd8498d94a8d247e6737cfe as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe 2024-11-09T17:27:49,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe, entries=150, sequenceid=38, filesize=11.7 K 2024-11-09T17:27:49,686 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d57ffb67cf090d609649a6e4d95584ca in 1238ms, sequenceid=38, compaction requested=false 2024-11-09T17:27:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=123}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=123 2024-11-09T17:27:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=123 2024-11-09T17:27:49,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=123, resume processing ppid=122 2024-11-09T17:27:49,688 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, ppid=122, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1540 sec 2024-11-09T17:27:49,689 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=122, table=TestAcidGuarantees in 2.1580 sec 2024-11-09T17:27:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:49,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:49,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:49,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1d807f68a98847f4b40c4f563a07ca42 is 50, key is test_row_0/A:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742223_1399 (size=16681) 2024-11-09T17:27:49,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173329820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173329821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173329823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173329824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173329927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173329927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173329927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173329929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173330133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173330133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173330133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173330134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1d807f68a98847f4b40c4f563a07ca42 2024-11-09T17:27:50,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/4040982793af497a801786b1fa765d8b is 50, key is test_row_0/B:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:50,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742224_1400 (size=12001) 2024-11-09T17:27:50,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/4040982793af497a801786b1fa765d8b 2024-11-09T17:27:50,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/53442d696f5b4ee2965b2a46c015bde3 is 50, key is test_row_0/C:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742225_1401 (size=12001) 2024-11-09T17:27:50,404 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:27:50,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173330438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173330440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173330440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173330445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/53442d696f5b4ee2965b2a46c015bde3 2024-11-09T17:27:50,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1d807f68a98847f4b40c4f563a07ca42 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42 2024-11-09T17:27:50,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42, entries=250, sequenceid=52, filesize=16.3 K 2024-11-09T17:27:50,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/4040982793af497a801786b1fa765d8b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b 2024-11-09T17:27:50,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T17:27:50,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/53442d696f5b4ee2965b2a46c015bde3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3 2024-11-09T17:27:50,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-09T17:27:50,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d57ffb67cf090d609649a6e4d95584ca in 844ms, sequenceid=52, compaction requested=true 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:50,636 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:50,636 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:50,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:50,637 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:50,637 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:50,637 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:27:50,637 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:27:50,637 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:50,637 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:50,637 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=35.2 K 2024-11-09T17:27:50,637 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=39.7 K 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting cf9fdbb3148f4fb99798287cc6800041, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731173267536 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f8823d2d6fc748cabdd17d02a7bb0ede, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731173267536 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 58b105c1677f448496155e6fd13f89b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731173267545 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 49554d7dc33d4957a882009c5e8118e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731173267545 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4040982793af497a801786b1fa765d8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268669 2024-11-09T17:27:50,638 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1d807f68a98847f4b40c4f563a07ca42, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268664 2024-11-09T17:27:50,643 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:50,644 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/828bf4622a82470985db97cb531fa554 is 50, key is test_row_0/B:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:50,652 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:50,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742226_1402 (size=12104) 2024-11-09T17:27:50,653 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/75358ddcbf7245efaee5752d4dcf3b56 is 50, key is test_row_0/A:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:50,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742227_1403 (size=12104) 2024-11-09T17:27:50,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:50,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:50,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:50,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a039a8b9fb74164a24d0347de7a45af is 50, key is test_row_0/A:col10/1731173270949/Put/seqid=0 2024-11-09T17:27:50,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742228_1404 (size=16681) 2024-11-09T17:27:50,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173330964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173330965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173330966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:50,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:50,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173330966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/828bf4622a82470985db97cb531fa554 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/828bf4622a82470985db97cb531fa554 2024-11-09T17:27:51,060 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into 828bf4622a82470985db97cb531fa554(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:51,060 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:51,060 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=13, startTime=1731173270636; duration=0sec 2024-11-09T17:27:51,060 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:51,060 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:27:51,060 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:51,061 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:51,061 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:27:51,061 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:51,061 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=35.2 K 2024-11-09T17:27:51,061 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting dabdcccba2b84a76abee03417195ee2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731173267536 2024-11-09T17:27:51,061 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a27fde54afd8498d94a8d247e6737cfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731173267545 2024-11-09T17:27:51,062 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 53442d696f5b4ee2965b2a46c015bde3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268669 2024-11-09T17:27:51,066 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#335 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:51,067 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/eb2e55b82ebb4134abba884c1e12b095 is 50, key is test_row_0/C:col10/1731173269792/Put/seqid=0 2024-11-09T17:27:51,069 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/75358ddcbf7245efaee5752d4dcf3b56 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/75358ddcbf7245efaee5752d4dcf3b56 2024-11-09T17:27:51,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742229_1405 (size=12104) 2024-11-09T17:27:51,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173331067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,074 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 75358ddcbf7245efaee5752d4dcf3b56(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:51,074 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:51,074 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=13, startTime=1731173270636; duration=0sec 2024-11-09T17:27:51,074 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:51,074 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:27:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173331073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173331073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173331075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173331272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173331279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173331280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,283 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173331280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a039a8b9fb74164a24d0347de7a45af 2024-11-09T17:27:51,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/df49250953cd458785f2f2d2c2cdf5f4 is 50, key is test_row_0/B:col10/1731173270949/Put/seqid=0 2024-11-09T17:27:51,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742230_1406 (size=12001) 2024-11-09T17:27:51,473 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/eb2e55b82ebb4134abba884c1e12b095 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/eb2e55b82ebb4134abba884c1e12b095 2024-11-09T17:27:51,477 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into eb2e55b82ebb4134abba884c1e12b095(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:51,477 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:51,477 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=13, startTime=1731173270636; duration=0sec 2024-11-09T17:27:51,477 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:51,477 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:27:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173331576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173331584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173331585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173331586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=122 2024-11-09T17:27:51,637 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 122 completed 2024-11-09T17:27:51,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees 2024-11-09T17:27:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:51,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:51,639 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=124, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:51,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:51,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173331690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,693 DEBUG [Thread-1786 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4146 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:51,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/df49250953cd458785f2f2d2c2cdf5f4 2024-11-09T17:27:51,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/342dfdf9091c45469603cd95c0781cf1 is 50, key is test_row_0/C:col10/1731173270949/Put/seqid=0 2024-11-09T17:27:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742231_1407 (size=12001) 2024-11-09T17:27:51,791 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-09T17:27:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:51,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:51,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:51,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:51,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:51,943 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:51,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-09T17:27:51,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:51,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:51,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:51,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:51,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:51,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:52,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:52,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173332080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:52,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173332090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173332092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:52,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173332092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,095 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-09T17:27:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:52,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] handler.RSProcedureHandler(58): pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:52,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=125 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=125 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:52,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/342dfdf9091c45469603cd95c0781cf1 2024-11-09T17:27:52,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a039a8b9fb74164a24d0347de7a45af as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af 2024-11-09T17:27:52,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af, entries=250, sequenceid=76, filesize=16.3 K 2024-11-09T17:27:52,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/df49250953cd458785f2f2d2c2cdf5f4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4 2024-11-09T17:27:52,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T17:27:52,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/342dfdf9091c45469603cd95c0781cf1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1 2024-11-09T17:27:52,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1, entries=150, sequenceid=76, filesize=11.7 K 2024-11-09T17:27:52,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for d57ffb67cf090d609649a6e4d95584ca in 1250ms, sequenceid=76, compaction requested=false 2024-11-09T17:27:52,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:52,247 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:52,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=125 2024-11-09T17:27:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:52,248 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-09T17:27:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:52,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:52,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:52,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/712f005833354b65810f84a8b44e85f4 is 50, key is test_row_0/A:col10/1731173270965/Put/seqid=0 2024-11-09T17:27:52,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742232_1408 (size=12001) 2024-11-09T17:27:52,659 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/712f005833354b65810f84a8b44e85f4 2024-11-09T17:27:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/11b0d2b6098045f7b25c31bc026f16ce is 50, key is test_row_0/B:col10/1731173270965/Put/seqid=0 2024-11-09T17:27:52,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742233_1409 (size=12001) 2024-11-09T17:27:52,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:53,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:53,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/11b0d2b6098045f7b25c31bc026f16ce 2024-11-09T17:27:53,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/de1cb803019a4d85959339bf854b91fe is 50, key is test_row_0/C:col10/1731173270965/Put/seqid=0 2024-11-09T17:27:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742234_1410 (size=12001) 2024-11-09T17:27:53,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173333126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173333126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173333128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173333129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173333232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173333232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173333236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173333238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173333435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173333435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173333441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173333445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,511 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/de1cb803019a4d85959339bf854b91fe 2024-11-09T17:27:53,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/712f005833354b65810f84a8b44e85f4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4 2024-11-09T17:27:53,518 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T17:27:53,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/11b0d2b6098045f7b25c31bc026f16ce as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce 2024-11-09T17:27:53,522 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T17:27:53,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/de1cb803019a4d85959339bf854b91fe as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe 2024-11-09T17:27:53,526 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe, entries=150, sequenceid=92, filesize=11.7 K 2024-11-09T17:27:53,527 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d57ffb67cf090d609649a6e4d95584ca in 1279ms, sequenceid=92, compaction requested=true 2024-11-09T17:27:53,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:53,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:53,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=125}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=125 2024-11-09T17:27:53,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=125 2024-11-09T17:27:53,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=125, resume processing ppid=124 2024-11-09T17:27:53,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=125, ppid=124, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-11-09T17:27:53,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=124, table=TestAcidGuarantees in 1.8910 sec 2024-11-09T17:27:53,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:53,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:53,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=124 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:53,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:53,742 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 124 completed 2024-11-09T17:27:53,743 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees 2024-11-09T17:27:53,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:53,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=126, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:53,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:53,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/5d38c5a1ea774733946a2c5c1db0f527 is 50, key is test_row_0/A:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:53,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742235_1411 (size=14341) 2024-11-09T17:27:53,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173333758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173333792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173333792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173333792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:53,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173333892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,896 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:53,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:53,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:53,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173333904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173333904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:53,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:53,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173333904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:54,049 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173334095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173334109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173334109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173334110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/5d38c5a1ea774733946a2c5c1db0f527 2024-11-09T17:27:54,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b9b929c18a2c4ae38c2c9ecd44425428 is 50, key is test_row_0/B:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742236_1412 (size=12001) 2024-11-09T17:27:54,201 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:54,353 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173334400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173334412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173334412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173334413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,505 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b9b929c18a2c4ae38c2c9ecd44425428 2024-11-09T17:27:54,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/9e21780421ef41198c0a34bac0d8a9bf is 50, key is test_row_0/C:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:54,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742237_1413 (size=12001) 2024-11-09T17:27:54,657 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,809 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:54,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173334906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173334918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173334918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173334921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,961 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:54,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:54,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] handler.RSProcedureHandler(58): pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=127 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=127 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:54,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/9e21780421ef41198c0a34bac0d8a9bf 2024-11-09T17:27:54,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/5d38c5a1ea774733946a2c5c1db0f527 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527 2024-11-09T17:27:54,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527, entries=200, sequenceid=116, filesize=14.0 K 2024-11-09T17:27:54,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b9b929c18a2c4ae38c2c9ecd44425428 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428 2024-11-09T17:27:54,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:27:54,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/9e21780421ef41198c0a34bac0d8a9bf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf 2024-11-09T17:27:54,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T17:27:54,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=73.80 KB/75570 for d57ffb67cf090d609649a6e4d95584ca in 1237ms, sequenceid=116, compaction requested=true 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:54,979 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:54,979 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:54,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55127 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:27:54,980 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,980 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:54,980 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/75358ddcbf7245efaee5752d4dcf3b56, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=53.8 K 2024-11-09T17:27:54,980 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/828bf4622a82470985db97cb531fa554, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=47.0 K 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 828bf4622a82470985db97cb531fa554, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268669 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 75358ddcbf7245efaee5752d4dcf3b56, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268669 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0a039a8b9fb74164a24d0347de7a45af, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731173269820 2024-11-09T17:27:54,980 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting df49250953cd458785f2f2d2c2cdf5f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731173269823 2024-11-09T17:27:54,981 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 712f005833354b65810f84a8b44e85f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731173270956 2024-11-09T17:27:54,981 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 11b0d2b6098045f7b25c31bc026f16ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731173270956 2024-11-09T17:27:54,981 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5d38c5a1ea774733946a2c5c1db0f527, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273124 2024-11-09T17:27:54,981 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting b9b929c18a2c4ae38c2c9ecd44425428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273125 2024-11-09T17:27:54,987 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:54,988 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:54,988 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/183b88db1241432497ab2b0e7d92a2c8 is 50, key is test_row_0/B:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:54,988 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/198493912cda4e489447403499057a5f is 50, key is test_row_0/A:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:54,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742239_1415 (size=12241) 2024-11-09T17:27:54,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742238_1414 (size=12241) 2024-11-09T17:27:54,996 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/198493912cda4e489447403499057a5f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/198493912cda4e489447403499057a5f 2024-11-09T17:27:54,996 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/183b88db1241432497ab2b0e7d92a2c8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/183b88db1241432497ab2b0e7d92a2c8 2024-11-09T17:27:55,000 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 198493912cda4e489447403499057a5f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:55,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:55,000 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=12, startTime=1731173274979; duration=0sec 2024-11-09T17:27:55,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:55,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:27:55,000 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:27:55,001 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into 183b88db1241432497ab2b0e7d92a2c8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:55,001 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:55,001 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=12, startTime=1731173274979; duration=0sec 2024-11-09T17:27:55,001 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:55,001 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:27:55,001 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:27:55,001 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:27:55,001 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:55,001 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/eb2e55b82ebb4134abba884c1e12b095, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=47.0 K 2024-11-09T17:27:55,002 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting eb2e55b82ebb4134abba884c1e12b095, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731173268669 2024-11-09T17:27:55,002 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 342dfdf9091c45469603cd95c0781cf1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731173269823 2024-11-09T17:27:55,002 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting de1cb803019a4d85959339bf854b91fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731173270956 2024-11-09T17:27:55,002 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9e21780421ef41198c0a34bac0d8a9bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273125 2024-11-09T17:27:55,010 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:55,010 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/ddf9ef5bdd4a45a6a2ad1470663c35e1 is 50, key is test_row_0/C:col10/1731173273128/Put/seqid=0 2024-11-09T17:27:55,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742240_1416 (size=12241) 2024-11-09T17:27:55,113 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=127 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:55,114 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:55,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:55,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4ec45457c6fb4602b815c40d78153acf is 50, key is test_row_0/A:col10/1731173273770/Put/seqid=0 2024-11-09T17:27:55,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742241_1417 (size=12051) 2024-11-09T17:27:55,419 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/ddf9ef5bdd4a45a6a2ad1470663c35e1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/ddf9ef5bdd4a45a6a2ad1470663c35e1 2024-11-09T17:27:55,423 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into ddf9ef5bdd4a45a6a2ad1470663c35e1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:55,423 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:55,423 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=12, startTime=1731173274979; duration=0sec 2024-11-09T17:27:55,423 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:55,423 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:27:55,521 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4ec45457c6fb4602b815c40d78153acf 2024-11-09T17:27:55,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/f5f680bbf2314ea8a18b425bee18f0ed is 50, key is test_row_0/B:col10/1731173273770/Put/seqid=0 2024-11-09T17:27:55,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742242_1418 (size=12051) 2024-11-09T17:27:55,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:55,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173335761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:55,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173335866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173335915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173335923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173335925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,932 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/f5f680bbf2314ea8a18b425bee18f0ed 2024-11-09T17:27:55,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/626e13cd1b4842f3bcb03f6e36c45dde is 50, key is test_row_0/C:col10/1731173273770/Put/seqid=0 2024-11-09T17:27:55,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:55,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173335933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:55,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742243_1419 (size=12051) 2024-11-09T17:27:56,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:56,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173336071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:56,341 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/626e13cd1b4842f3bcb03f6e36c45dde 2024-11-09T17:27:56,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4ec45457c6fb4602b815c40d78153acf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf 2024-11-09T17:27:56,347 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T17:27:56,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/f5f680bbf2314ea8a18b425bee18f0ed as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed 2024-11-09T17:27:56,350 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T17:27:56,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/626e13cd1b4842f3bcb03f6e36c45dde as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde 2024-11-09T17:27:56,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T17:27:56,354 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d57ffb67cf090d609649a6e4d95584ca in 1240ms, sequenceid=132, compaction requested=false 2024-11-09T17:27:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:56,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=127}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=127 2024-11-09T17:27:56,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=127 2024-11-09T17:27:56,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=127, resume processing ppid=126 2024-11-09T17:27:56,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, ppid=126, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6100 sec 2024-11-09T17:27:56,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=126, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=126, table=TestAcidGuarantees in 2.6130 sec 2024-11-09T17:27:56,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:56,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:56,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:56,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/220e4928f329403c9a8b5e2782a0b135 is 50, key is test_row_0/A:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:56,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742244_1420 (size=14541) 2024-11-09T17:27:56,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/220e4928f329403c9a8b5e2782a0b135 2024-11-09T17:27:56,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/d73ba37bf6844b8a976362167d08036e is 50, key is test_row_0/B:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:56,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742245_1421 (size=12151) 2024-11-09T17:27:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173336418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:56,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:56,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173336522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:56,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173336727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:56,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/d73ba37bf6844b8a976362167d08036e 2024-11-09T17:27:56,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/360065a0b1b845128c7870581acad11e is 50, key is test_row_0/C:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:56,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742246_1422 (size=12151) 2024-11-09T17:27:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173337030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,205 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/360065a0b1b845128c7870581acad11e 2024-11-09T17:27:57,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/220e4928f329403c9a8b5e2782a0b135 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135 2024-11-09T17:27:57,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135, entries=200, sequenceid=156, filesize=14.2 K 2024-11-09T17:27:57,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/d73ba37bf6844b8a976362167d08036e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e 2024-11-09T17:27:57,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T17:27:57,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/360065a0b1b845128c7870581acad11e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e 2024-11-09T17:27:57,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T17:27:57,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d57ffb67cf090d609649a6e4d95584ca in 840ms, sequenceid=156, compaction requested=true 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:27:57,219 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:57,219 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:27:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38833 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:27:57,220 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:57,220 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:57,220 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/198493912cda4e489447403499057a5f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=37.9 K 2024-11-09T17:27:57,220 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/183b88db1241432497ab2b0e7d92a2c8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=35.6 K 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 183b88db1241432497ab2b0e7d92a2c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273125 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 198493912cda4e489447403499057a5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273125 2024-11-09T17:27:57,220 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f5f680bbf2314ea8a18b425bee18f0ed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173273749 2024-11-09T17:27:57,221 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4ec45457c6fb4602b815c40d78153acf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173273749 2024-11-09T17:27:57,221 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting d73ba37bf6844b8a976362167d08036e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:27:57,221 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 220e4928f329403c9a8b5e2782a0b135, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:27:57,227 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:57,227 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:57,228 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/2b631356810646598c967fd9dcb9c586 is 50, key is test_row_0/B:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:57,228 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4a294a1e71c64a88948a8e6bd88bd6fe is 50, key is test_row_0/A:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:57,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742248_1424 (size=12493) 2024-11-09T17:27:57,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742247_1423 (size=12493) 2024-11-09T17:27:57,239 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4a294a1e71c64a88948a8e6bd88bd6fe as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4a294a1e71c64a88948a8e6bd88bd6fe 2024-11-09T17:27:57,244 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 4a294a1e71c64a88948a8e6bd88bd6fe(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:57,244 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:57,244 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=13, startTime=1731173277219; duration=0sec 2024-11-09T17:27:57,244 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:27:57,244 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:27:57,244 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:27:57,245 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36443 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:27:57,245 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:27:57,245 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:57,245 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/ddf9ef5bdd4a45a6a2ad1470663c35e1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=35.6 K 2024-11-09T17:27:57,245 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting ddf9ef5bdd4a45a6a2ad1470663c35e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731173273125 2024-11-09T17:27:57,245 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 626e13cd1b4842f3bcb03f6e36c45dde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173273749 2024-11-09T17:27:57,246 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 360065a0b1b845128c7870581acad11e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:27:57,252 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#355 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:27:57,252 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/9e1754269653454bb23d4e1f8c6e4f4d is 50, key is test_row_0/C:col10/1731173275745/Put/seqid=0 2024-11-09T17:27:57,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742249_1425 (size=12493) 2024-11-09T17:27:57,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:57,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:57,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:57,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 is 50, key is test_row_0/A:col10/1731173276417/Put/seqid=0 2024-11-09T17:27:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742250_1426 (size=14541) 2024-11-09T17:27:57,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173337612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,641 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/2b631356810646598c967fd9dcb9c586 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/2b631356810646598c967fd9dcb9c586 2024-11-09T17:27:57,644 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into 2b631356810646598c967fd9dcb9c586(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:57,644 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:57,644 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=13, startTime=1731173277219; duration=0sec 2024-11-09T17:27:57,644 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:57,644 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:27:57,695 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/9e1754269653454bb23d4e1f8c6e4f4d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e1754269653454bb23d4e1f8c6e4f4d 2024-11-09T17:27:57,698 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into 9e1754269653454bb23d4e1f8c6e4f4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:27:57,698 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:57,698 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=13, startTime=1731173277219; duration=0sec 2024-11-09T17:27:57,698 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:27:57,698 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:27:57,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173337716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-09T17:27:57,849 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 126 completed 2024-11-09T17:27:57,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:57,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees 2024-11-09T17:27:57,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:57,852 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:57,853 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=128, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:57,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:57,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173337920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173337928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,931 DEBUG [Thread-1790 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4172 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:57,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173337931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,934 DEBUG [Thread-1784 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4143 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:57,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173337936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,940 DEBUG [Thread-1788 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4149 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:57,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 2024-11-09T17:27:57,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/594f4d86b07b4bbb8ae8316a6ddb8dfb is 50, key is test_row_0/B:col10/1731173276417/Put/seqid=0 2024-11-09T17:27:57,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:57,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742251_1427 (size=12151) 2024-11-09T17:27:57,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:57,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173337955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:57,959 DEBUG [Thread-1782 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4168 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:27:58,004 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:58,156 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173338225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,308 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/594f4d86b07b4bbb8ae8316a6ddb8dfb 2024-11-09T17:27:58,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/4232b2f9716e424e992132f665f4f4b0 is 50, key is test_row_0/C:col10/1731173276417/Put/seqid=0 2024-11-09T17:27:58,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742252_1428 (size=12151) 2024-11-09T17:27:58,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:58,461 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,612 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173338733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/4232b2f9716e424e992132f665f4f4b0 2024-11-09T17:27:58,765 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:27:58,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] handler.RSProcedureHandler(58): pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=129 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=129 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:27:58,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 2024-11-09T17:27:58,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1, entries=200, sequenceid=171, filesize=14.2 K 2024-11-09T17:27:58,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/594f4d86b07b4bbb8ae8316a6ddb8dfb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb 2024-11-09T17:27:58,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T17:27:58,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/4232b2f9716e424e992132f665f4f4b0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0 2024-11-09T17:27:58,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T17:27:58,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d57ffb67cf090d609649a6e4d95584ca in 1245ms, sequenceid=171, compaction requested=false 2024-11-09T17:27:58,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:58,923 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:27:58,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=129 2024-11-09T17:27:58,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,923 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-09T17:27:58,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:58,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:58,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/6eaab712e1e6440b93faefa99526ef07 is 50, key is test_row_0/A:col10/1731173277606/Put/seqid=0 2024-11-09T17:27:58,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742253_1429 (size=12151) 2024-11-09T17:27:58,930 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/6eaab712e1e6440b93faefa99526ef07 2024-11-09T17:27:58,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea is 50, key is test_row_0/B:col10/1731173277606/Put/seqid=0 2024-11-09T17:27:58,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742254_1430 (size=12151) 2024-11-09T17:27:58,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea 2024-11-09T17:27:58,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/df496940bd774fabb53acce968c71020 is 50, key is test_row_0/C:col10/1731173277606/Put/seqid=0 2024-11-09T17:27:58,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742255_1431 (size=12151) 2024-11-09T17:27:58,948 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/df496940bd774fabb53acce968c71020 2024-11-09T17:27:58,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/6eaab712e1e6440b93faefa99526ef07 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07 2024-11-09T17:27:58,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:27:58,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:58,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea 2024-11-09T17:27:58,957 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:27:58,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/df496940bd774fabb53acce968c71020 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020 2024-11-09T17:27:58,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020, entries=150, sequenceid=195, filesize=11.9 K 2024-11-09T17:27:58,961 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(3040): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for d57ffb67cf090d609649a6e4d95584ca in 37ms, sequenceid=195, compaction requested=true 2024-11-09T17:27:58,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:27:58,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:27:58,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-11-09T17:27:58,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=129 2024-11-09T17:27:58,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=129, resume processing ppid=128 2024-11-09T17:27:58,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=129, ppid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1080 sec 2024-11-09T17:27:58,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=128, table=TestAcidGuarantees in 1.1130 sec 2024-11-09T17:27:59,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:27:59,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:27:59,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:27:59,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/3ad8c749422c4181b0b62b9836e4ae6e is 50, key is test_row_0/A:col10/1731173279758/Put/seqid=0 2024-11-09T17:27:59,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742256_1432 (size=16931) 2024-11-09T17:27:59,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:59,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173339860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:27:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=128 2024-11-09T17:27:59,955 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 128 completed 2024-11-09T17:27:59,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:27:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees 2024-11-09T17:27:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:27:59,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:27:59,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=130, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:27:59,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:27:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:27:59,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173339969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:00,108 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/3ad8c749422c4181b0b62b9836e4ae6e 2024-11-09T17:28:00,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/c08c8b4e95834c07a68f421c9e340377 is 50, key is test_row_0/B:col10/1731173279758/Put/seqid=0 2024-11-09T17:28:00,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173340176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742257_1433 (size=12151) 2024-11-09T17:28:00,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:00,261 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,413 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173340482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:00,565 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/c08c8b4e95834c07a68f421c9e340377 2024-11-09T17:28:00,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/acff3df39d5a415c8fe400b1335e045a is 50, key is test_row_0/C:col10/1731173279758/Put/seqid=0 2024-11-09T17:28:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742258_1434 (size=12151) 2024-11-09T17:28:00,717 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,870 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:00,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] handler.RSProcedureHandler(58): pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=131 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=131 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:00,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/acff3df39d5a415c8fe400b1335e045a 2024-11-09T17:28:00,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:00,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173340987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:00,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/3ad8c749422c4181b0b62b9836e4ae6e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e 2024-11-09T17:28:00,996 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e, entries=250, sequenceid=206, filesize=16.5 K 2024-11-09T17:28:00,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/c08c8b4e95834c07a68f421c9e340377 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377 2024-11-09T17:28:01,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T17:28:01,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/acff3df39d5a415c8fe400b1335e045a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a 2024-11-09T17:28:01,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T17:28:01,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d57ffb67cf090d609649a6e4d95584ca in 1243ms, sequenceid=206, compaction requested=true 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:01,004 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:01,004 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:01,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:01,005 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56116 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:01,005 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:01,005 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:28:01,005 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:28:01,005 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:01,005 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:01,005 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/2b631356810646598c967fd9dcb9c586, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=47.8 K 2024-11-09T17:28:01,005 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4a294a1e71c64a88948a8e6bd88bd6fe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=54.8 K 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4a294a1e71c64a88948a8e6bd88bd6fe, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2b631356810646598c967fd9dcb9c586, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 594f4d86b07b4bbb8ae8316a6ddb8dfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731173276402 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting ddcfa3ad03284fdbb60dbdfd4c026dc1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731173276396 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting fd8c7838d43c4c1cb89f2be7d3d8d2ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173277599 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6eaab712e1e6440b93faefa99526ef07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173277599 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c08c8b4e95834c07a68f421c9e340377, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:01,006 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3ad8c749422c4181b0b62b9836e4ae6e, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:01,019 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#365 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:01,020 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/ab05546196f94fb7a17fee1eb2dcc036 is 50, key is test_row_0/B:col10/1731173279758/Put/seqid=0 2024-11-09T17:28:01,020 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:01,020 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4728a90091ae49489c63edbd8de85184 is 50, key is test_row_0/A:col10/1731173279758/Put/seqid=0 2024-11-09T17:28:01,022 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:01,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=131 2024-11-09T17:28:01,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:01,022 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:01,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742259_1435 (size=12629) 2024-11-09T17:28:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742260_1436 (size=12629) 2024-11-09T17:28:01,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/aacf345df2aa457c8ea6dcef98b7ad72 is 50, key is test_row_0/A:col10/1731173279845/Put/seqid=0 2024-11-09T17:28:01,029 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/ab05546196f94fb7a17fee1eb2dcc036 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ab05546196f94fb7a17fee1eb2dcc036 2024-11-09T17:28:01,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742261_1437 (size=12151) 2024-11-09T17:28:01,033 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into ab05546196f94fb7a17fee1eb2dcc036(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:01,033 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:01,033 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=12, startTime=1731173281004; duration=0sec 2024-11-09T17:28:01,033 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:01,033 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:28:01,033 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:01,034 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:01,034 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:28:01,034 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:01,034 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e1754269653454bb23d4e1f8c6e4f4d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=47.8 K 2024-11-09T17:28:01,034 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 9e1754269653454bb23d4e1f8c6e4f4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731173275745 2024-11-09T17:28:01,035 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 4232b2f9716e424e992132f665f4f4b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731173276402 2024-11-09T17:28:01,035 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting df496940bd774fabb53acce968c71020, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731173277599 2024-11-09T17:28:01,035 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting acff3df39d5a415c8fe400b1335e045a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:01,041 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:01,041 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/44193fe7fc974b448feb7f0e3dfc1d2e is 50, key is test_row_0/C:col10/1731173279758/Put/seqid=0 2024-11-09T17:28:01,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742262_1438 (size=12629) 2024-11-09T17:28:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:01,429 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/4728a90091ae49489c63edbd8de85184 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4728a90091ae49489c63edbd8de85184 2024-11-09T17:28:01,431 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/aacf345df2aa457c8ea6dcef98b7ad72 2024-11-09T17:28:01,433 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 4728a90091ae49489c63edbd8de85184(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:01,433 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:01,433 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=12, startTime=1731173281004; duration=0sec 2024-11-09T17:28:01,433 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:01,433 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:28:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/886110fba0b44604b5e3c0557938dfcf is 50, key is test_row_0/B:col10/1731173279845/Put/seqid=0 2024-11-09T17:28:01,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742263_1439 (size=12151) 2024-11-09T17:28:01,449 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/44193fe7fc974b448feb7f0e3dfc1d2e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/44193fe7fc974b448feb7f0e3dfc1d2e 2024-11-09T17:28:01,452 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into 44193fe7fc974b448feb7f0e3dfc1d2e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:01,452 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:01,452 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=12, startTime=1731173281004; duration=0sec 2024-11-09T17:28:01,452 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:01,452 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:28:01,841 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/886110fba0b44604b5e3c0557938dfcf 2024-11-09T17:28:01,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/bd319f07afb04b3dba4462b631adf750 is 50, key is test_row_0/C:col10/1731173279845/Put/seqid=0 2024-11-09T17:28:01,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742264_1440 (size=12151) 2024-11-09T17:28:01,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:01,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173341953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:01,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:01,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173341955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36610 deadline: 1731173341963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:01,967 DEBUG [Thread-1782 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8175 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:01,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36602 deadline: 1731173341971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:01,971 DEBUG [Thread-1784 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8180 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:01,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173341994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173342057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173342058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,252 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/bd319f07afb04b3dba4462b631adf750 2024-11-09T17:28:02,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/aacf345df2aa457c8ea6dcef98b7ad72 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72 2024-11-09T17:28:02,258 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T17:28:02,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/886110fba0b44604b5e3c0557938dfcf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf 2024-11-09T17:28:02,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T17:28:02,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/bd319f07afb04b3dba4462b631adf750 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750 2024-11-09T17:28:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173342260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750, entries=150, sequenceid=231, filesize=11.9 K 2024-11-09T17:28:02,265 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d57ffb67cf090d609649a6e4d95584ca in 1243ms, sequenceid=231, compaction requested=false 2024-11-09T17:28:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:02,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-11-09T17:28:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-11-09T17:28:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:02,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:02,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:02,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-11-09T17:28:02,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3090 sec 2024-11-09T17:28:02,269 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=130, table=TestAcidGuarantees in 2.3120 sec 2024-11-09T17:28:02,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/332b16ab72344c339dbf2bb6f1bac8fa is 50, key is test_row_0/A:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:02,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742265_1441 (size=14541) 2024-11-09T17:28:02,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173342361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173342467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173342564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/332b16ab72344c339dbf2bb6f1bac8fa 2024-11-09T17:28:02,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173342672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:02,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/5381327495914db2ba9fc30b2bd4cf56 is 50, key is test_row_0/B:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:02,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742266_1442 (size=12151) 2024-11-09T17:28:02,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173342978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:03,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:03,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173343073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:03,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/5381327495914db2ba9fc30b2bd4cf56 2024-11-09T17:28:03,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/3598da3205aa470b91d1596cfd71e193 is 50, key is test_row_0/C:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:03,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742267_1443 (size=12151) 2024-11-09T17:28:03,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173343481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:03,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/3598da3205aa470b91d1596cfd71e193 2024-11-09T17:28:03,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/332b16ab72344c339dbf2bb6f1bac8fa as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa 2024-11-09T17:28:03,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa, entries=200, sequenceid=246, filesize=14.2 K 2024-11-09T17:28:03,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/5381327495914db2ba9fc30b2bd4cf56 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56 2024-11-09T17:28:03,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56, entries=150, sequenceid=246, filesize=11.9 K 2024-11-09T17:28:03,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/3598da3205aa470b91d1596cfd71e193 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193 2024-11-09T17:28:03,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193, entries=150, sequenceid=246, filesize=11.9 K 2024-11-09T17:28:03,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d57ffb67cf090d609649a6e4d95584ca in 1278ms, sequenceid=246, compaction requested=true 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:03,545 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:03,545 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:28:03,546 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:03,546 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ab05546196f94fb7a17fee1eb2dcc036, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=36.1 K 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39321 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:28:03,546 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:03,546 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4728a90091ae49489c63edbd8de85184, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=38.4 K 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ab05546196f94fb7a17fee1eb2dcc036, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:03,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 4728a90091ae49489c63edbd8de85184, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:03,547 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 886110fba0b44604b5e3c0557938dfcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173279845 2024-11-09T17:28:03,547 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting aacf345df2aa457c8ea6dcef98b7ad72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173279845 2024-11-09T17:28:03,547 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 5381327495914db2ba9fc30b2bd4cf56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281945 2024-11-09T17:28:03,547 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 332b16ab72344c339dbf2bb6f1bac8fa, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281941 2024-11-09T17:28:03,552 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:03,552 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1426ecdea633430f850dddb61107ac29 is 50, key is test_row_0/A:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:03,559 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#375 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:03,559 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/ea0474ba59794a4e9bb6c034ddc6f0c7 is 50, key is test_row_0/B:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:03,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742268_1444 (size=12731) 2024-11-09T17:28:03,564 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1426ecdea633430f850dddb61107ac29 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1426ecdea633430f850dddb61107ac29 2024-11-09T17:28:03,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742269_1445 (size=12731) 2024-11-09T17:28:03,569 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 1426ecdea633430f850dddb61107ac29(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:03,569 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:03,569 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=13, startTime=1731173283545; duration=0sec 2024-11-09T17:28:03,569 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:03,569 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:28:03,569 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:03,570 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:03,570 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:28:03,570 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:03,570 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/44193fe7fc974b448feb7f0e3dfc1d2e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=36.1 K 2024-11-09T17:28:03,570 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 44193fe7fc974b448feb7f0e3dfc1d2e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731173279750 2024-11-09T17:28:03,571 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting bd319f07afb04b3dba4462b631adf750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731173279845 2024-11-09T17:28:03,571 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3598da3205aa470b91d1596cfd71e193, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281945 2024-11-09T17:28:03,572 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/ea0474ba59794a4e9bb6c034ddc6f0c7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ea0474ba59794a4e9bb6c034ddc6f0c7 2024-11-09T17:28:03,576 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into ea0474ba59794a4e9bb6c034ddc6f0c7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:03,576 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:03,576 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=13, startTime=1731173283545; duration=0sec 2024-11-09T17:28:03,576 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:03,576 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:28:03,577 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#376 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:03,577 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/f423a8a771b74db2bd84a38b55197595 is 50, key is test_row_0/C:col10/1731173282266/Put/seqid=0 2024-11-09T17:28:03,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742270_1446 (size=12731) 2024-11-09T17:28:03,987 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/f423a8a771b74db2bd84a38b55197595 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/f423a8a771b74db2bd84a38b55197595 2024-11-09T17:28:03,990 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into f423a8a771b74db2bd84a38b55197595(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:03,990 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:03,990 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=13, startTime=1731173283545; duration=0sec 2024-11-09T17:28:03,990 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:03,990 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:28:04,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:04,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/533160cf15404241ab8c894fdbd41872 is 50, key is test_row_0/A:col10/1731173282357/Put/seqid=0 2024-11-09T17:28:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742271_1447 (size=12301) 2024-11-09T17:28:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173344036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-09T17:28:04,061 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 130 completed 2024-11-09T17:28:04,061 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees 2024-11-09T17:28:04,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:04,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:04,063 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=132, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:04,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:04,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173344079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173344140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:04,214 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:04,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173344346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:04,366 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/533160cf15404241ab8c894fdbd41872 2024-11-09T17:28:04,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/94869c0bfb1746e5a938493f6d680a2f is 50, key is test_row_0/B:col10/1731173282357/Put/seqid=0 2024-11-09T17:28:04,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742272_1448 (size=12301) 2024-11-09T17:28:04,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173344483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,519 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:04,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173344651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:04,671 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,823 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] handler.RSProcedureHandler(58): pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=133 java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=133 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:04,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/94869c0bfb1746e5a938493f6d680a2f 2024-11-09T17:28:04,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/106a943be05c4aef9a2044d2d0897deb is 50, key is test_row_0/C:col10/1731173282357/Put/seqid=0 2024-11-09T17:28:04,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742273_1449 (size=12301) 2024-11-09T17:28:04,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/106a943be05c4aef9a2044d2d0897deb 2024-11-09T17:28:04,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/533160cf15404241ab8c894fdbd41872 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872 2024-11-09T17:28:04,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T17:28:04,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/94869c0bfb1746e5a938493f6d680a2f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f 2024-11-09T17:28:04,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T17:28:04,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/106a943be05c4aef9a2044d2d0897deb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb 2024-11-09T17:28:04,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T17:28:04,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d57ffb67cf090d609649a6e4d95584ca in 850ms, sequenceid=274, compaction requested=false 2024-11-09T17:28:04,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:04,975 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:04,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=133 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:04,976 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:04,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/dc51ea18a6b542d3aa9281610a8826af is 50, key is test_row_0/A:col10/1731173284035/Put/seqid=0 2024-11-09T17:28:04,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742274_1450 (size=9857) 2024-11-09T17:28:05,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:05,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. as already flushing 2024-11-09T17:28:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:05,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173345246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:05,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173345351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:05,383 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/dc51ea18a6b542d3aa9281610a8826af 2024-11-09T17:28:05,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/777c671f1931462a8de22f346764bd0e is 50, key is test_row_0/B:col10/1731173284035/Put/seqid=0 2024-11-09T17:28:05,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742275_1451 (size=9857) 2024-11-09T17:28:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173345555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:05,792 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/777c671f1931462a8de22f346764bd0e 2024-11-09T17:28:05,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a16b60581ff94e91995353f21bc90af3 is 50, key is test_row_0/C:col10/1731173284035/Put/seqid=0 2024-11-09T17:28:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742276_1452 (size=9857) 2024-11-09T17:28:05,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173345859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36612 deadline: 1731173346094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,098 DEBUG [Thread-1788 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4144 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:06,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:06,200 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a16b60581ff94e91995353f21bc90af3 2024-11-09T17:28:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/dc51ea18a6b542d3aa9281610a8826af as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af 2024-11-09T17:28:06,206 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af, entries=100, sequenceid=285, filesize=9.6 K 2024-11-09T17:28:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/777c671f1931462a8de22f346764bd0e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e 2024-11-09T17:28:06,209 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e, entries=100, sequenceid=285, filesize=9.6 K 2024-11-09T17:28:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/a16b60581ff94e91995353f21bc90af3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3 2024-11-09T17:28:06,213 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3, entries=100, sequenceid=285, filesize=9.6 K 2024-11-09T17:28:06,213 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d57ffb67cf090d609649a6e4d95584ca in 1237ms, sequenceid=285, compaction requested=true 2024-11-09T17:28:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:06,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=133}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=133 2024-11-09T17:28:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=133 2024-11-09T17:28:06,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=133, resume processing ppid=132 2024-11-09T17:28:06,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, ppid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-11-09T17:28:06,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=132, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=132, table=TestAcidGuarantees in 2.1550 sec 2024-11-09T17:28:06,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:06,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:06,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:06,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/9ca9c4da0b7e4e3dbdf781d577c52394 is 50, key is test_row_0/A:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:06,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742277_1453 (size=14741) 2024-11-09T17:28:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173346413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36642 deadline: 1731173346490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,493 DEBUG [Thread-1790 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4132 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_2' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., hostname=c28fd47d982f,45633,1731173138129, seqNum=2, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173346524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,582 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:28:06,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:06,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173346731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:06,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/9ca9c4da0b7e4e3dbdf781d577c52394 2024-11-09T17:28:06,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/65d3e24845a84d0f98d2236ecb68ffe0 is 50, key is test_row_0/B:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:06,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742278_1454 (size=12301) 2024-11-09T17:28:07,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173347036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:07,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/65d3e24845a84d0f98d2236ecb68ffe0 2024-11-09T17:28:07,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2d2c10f43f0a44699fd2ea8d1b6bb923 is 50, key is test_row_0/C:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:07,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742279_1455 (size=12301) 2024-11-09T17:28:07,531 DEBUG [Thread-1797 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3652e74d to 127.0.0.1:57222 2024-11-09T17:28:07,531 DEBUG [Thread-1797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:07,534 DEBUG [Thread-1795 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x496fe03f to 127.0.0.1:57222 2024-11-09T17:28:07,534 DEBUG [Thread-1795 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:07,534 DEBUG [Thread-1801 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x73d92042 to 127.0.0.1:57222 2024-11-09T17:28:07,534 DEBUG [Thread-1801 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:07,537 DEBUG [Thread-1799 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2405c04e to 127.0.0.1:57222 2024-11-09T17:28:07,537 DEBUG [Thread-1799 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:07,539 DEBUG [Thread-1793 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7cf40102 to 127.0.0.1:57222 2024-11-09T17:28:07,539 DEBUG [Thread-1793 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:07,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:07,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36682 deadline: 1731173347543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:07,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2d2c10f43f0a44699fd2ea8d1b6bb923 2024-11-09T17:28:07,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/9ca9c4da0b7e4e3dbdf781d577c52394 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394 2024-11-09T17:28:07,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394, entries=200, sequenceid=311, filesize=14.4 K 2024-11-09T17:28:07,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/65d3e24845a84d0f98d2236ecb68ffe0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0 2024-11-09T17:28:07,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0, entries=150, sequenceid=311, filesize=12.0 K 2024-11-09T17:28:07,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2d2c10f43f0a44699fd2ea8d1b6bb923 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923 2024-11-09T17:28:07,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923, entries=150, sequenceid=311, filesize=12.0 K 2024-11-09T17:28:07,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d57ffb67cf090d609649a6e4d95584ca in 1248ms, sequenceid=311, compaction requested=true 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:07,623 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d57ffb67cf090d609649a6e4d95584ca:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:07,623 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:07,624 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49630 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:07,624 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/A is initiating minor compaction (all files) 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/B is initiating minor compaction (all files) 2024-11-09T17:28:07,625 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/A in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:07,625 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/B in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:07,625 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ea0474ba59794a4e9bb6c034ddc6f0c7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=46.1 K 2024-11-09T17:28:07,625 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1426ecdea633430f850dddb61107ac29, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=48.5 K 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1426ecdea633430f850dddb61107ac29, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281945 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ea0474ba59794a4e9bb6c034ddc6f0c7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281945 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 94869c0bfb1746e5a938493f6d680a2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173282318 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 533160cf15404241ab8c894fdbd41872, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173282318 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 777c671f1931462a8de22f346764bd0e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731173284031 2024-11-09T17:28:07,625 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting dc51ea18a6b542d3aa9281610a8826af, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731173284031 2024-11-09T17:28:07,626 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 9ca9c4da0b7e4e3dbdf781d577c52394, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731173285233 2024-11-09T17:28:07,626 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 65d3e24845a84d0f98d2236ecb68ffe0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731173285243 2024-11-09T17:28:07,631 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:07,631 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#B#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:07,632 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/4ffeda699f3e4e57897992fac24f0726 is 50, key is test_row_0/B:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:07,632 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/98cecaae41254e83a673f33cde593114 is 50, key is test_row_0/A:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:07,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742281_1457 (size=13017) 2024-11-09T17:28:07,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742280_1456 (size=13017) 2024-11-09T17:28:08,044 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/4ffeda699f3e4e57897992fac24f0726 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4ffeda699f3e4e57897992fac24f0726 2024-11-09T17:28:08,046 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/98cecaae41254e83a673f33cde593114 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/98cecaae41254e83a673f33cde593114 2024-11-09T17:28:08,050 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/B of d57ffb67cf090d609649a6e4d95584ca into 4ffeda699f3e4e57897992fac24f0726(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:08,050 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/B, priority=12, startTime=1731173287623; duration=0sec 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:08,050 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/A of d57ffb67cf090d609649a6e4d95584ca into 98cecaae41254e83a673f33cde593114(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:B 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:08,050 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/A, priority=12, startTime=1731173287623; duration=0sec 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:08,050 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:A 2024-11-09T17:28:08,052 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47190 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:08,052 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): d57ffb67cf090d609649a6e4d95584ca/C is initiating minor compaction (all files) 2024-11-09T17:28:08,052 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of d57ffb67cf090d609649a6e4d95584ca/C in TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:08,052 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/f423a8a771b74db2bd84a38b55197595, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp, totalSize=46.1 K 2024-11-09T17:28:08,052 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f423a8a771b74db2bd84a38b55197595, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731173281945 2024-11-09T17:28:08,053 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 106a943be05c4aef9a2044d2d0897deb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731173282318 2024-11-09T17:28:08,053 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting a16b60581ff94e91995353f21bc90af3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731173284031 2024-11-09T17:28:08,054 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2d2c10f43f0a44699fd2ea8d1b6bb923, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731173285243 2024-11-09T17:28:08,064 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d57ffb67cf090d609649a6e4d95584ca#C#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:08,064 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/18bb699ed22c4773943a2fd3aa968072 is 50, key is test_row_0/C:col10/1731173285245/Put/seqid=0 2024-11-09T17:28:08,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742282_1458 (size=13017) 2024-11-09T17:28:08,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-09T17:28:08,167 INFO [Thread-1792 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 132 completed 2024-11-09T17:28:08,479 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/18bb699ed22c4773943a2fd3aa968072 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/18bb699ed22c4773943a2fd3aa968072 2024-11-09T17:28:08,486 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in d57ffb67cf090d609649a6e4d95584ca/C of d57ffb67cf090d609649a6e4d95584ca into 18bb699ed22c4773943a2fd3aa968072(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:08,486 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:08,486 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca., storeName=d57ffb67cf090d609649a6e4d95584ca/C, priority=12, startTime=1731173287623; duration=0sec 2024-11-09T17:28:08,486 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:08,486 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d57ffb67cf090d609649a6e4d95584ca:C 2024-11-09T17:28:08,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:08,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:28:08,556 DEBUG [Thread-1786 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4414259d to 127.0.0.1:57222 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:08,557 DEBUG [Thread-1786 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:08,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:08,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1e513fc452ac49f783d30c0e9548ae95 is 50, key is test_row_0/A:col10/1731173286411/Put/seqid=0 2024-11-09T17:28:08,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742283_1459 (size=12301) 2024-11-09T17:28:08,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1e513fc452ac49f783d30c0e9548ae95 2024-11-09T17:28:08,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/1fda9e4d1c16438cb968521852f3ca97 is 50, key is test_row_0/B:col10/1731173286411/Put/seqid=0 2024-11-09T17:28:08,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742284_1460 (size=12301) 2024-11-09T17:28:09,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/1fda9e4d1c16438cb968521852f3ca97 2024-11-09T17:28:09,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/5da05a064a594efd86d120d57dfc920f is 50, key is test_row_0/C:col10/1731173286411/Put/seqid=0 2024-11-09T17:28:09,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742285_1461 (size=12301) 2024-11-09T17:28:09,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/5da05a064a594efd86d120d57dfc920f 2024-11-09T17:28:09,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/1e513fc452ac49f783d30c0e9548ae95 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1e513fc452ac49f783d30c0e9548ae95 2024-11-09T17:28:09,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1e513fc452ac49f783d30c0e9548ae95, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T17:28:09,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/1fda9e4d1c16438cb968521852f3ca97 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/1fda9e4d1c16438cb968521852f3ca97 2024-11-09T17:28:09,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/1fda9e4d1c16438cb968521852f3ca97, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T17:28:09,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/5da05a064a594efd86d120d57dfc920f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/5da05a064a594efd86d120d57dfc920f 2024-11-09T17:28:09,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/5da05a064a594efd86d120d57dfc920f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-09T17:28:09,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for d57ffb67cf090d609649a6e4d95584ca in 1274ms, sequenceid=325, compaction requested=false 2024-11-09T17:28:09,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:10,112 DEBUG [Thread-1788 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ed69825 to 127.0.0.1:57222 2024-11-09T17:28:10,113 DEBUG [Thread-1788 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:10,499 DEBUG [Thread-1790 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x11193a0c to 127.0.0.1:57222 2024-11-09T17:28:10,499 DEBUG [Thread-1790 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:12,000 DEBUG [Thread-1782 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f142b04 to 127.0.0.1:57222 2024-11-09T17:28:12,000 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:12,050 DEBUG [Thread-1784 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0de9f076 to 127.0.0.1:57222 2024-11-09T17:28:12,050 DEBUG [Thread-1784 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2369 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7107 rows 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2378 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7134 rows 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2383 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7149 rows 2024-11-09T17:28:12,050 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2375 2024-11-09T17:28:12,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7125 rows 2024-11-09T17:28:12,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 2382 2024-11-09T17:28:12,051 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 7146 rows 2024-11-09T17:28:12,051 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:28:12,051 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x63cefe40 to 127.0.0.1:57222 2024-11-09T17:28:12,051 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:12,055 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:28:12,055 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:28:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=134, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:12,058 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173292058"}]},"ts":"1731173292058"} 2024-11-09T17:28:12,059 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:28:12,078 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:28:12,079 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:28:12,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, UNASSIGN}] 2024-11-09T17:28:12,081 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=136, ppid=135, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, UNASSIGN 2024-11-09T17:28:12,082 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=136 updating hbase:meta row=d57ffb67cf090d609649a6e4d95584ca, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:12,083 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:28:12,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE; CloseRegionProcedure d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:28:12,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:12,235 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:12,236 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] handler.UnassignRegionHandler(124): Close d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:12,237 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:28:12,237 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1681): Closing d57ffb67cf090d609649a6e4d95584ca, disabling compactions & flushes 2024-11-09T17:28:12,237 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:12,237 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:12,237 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. after waiting 0 ms 2024-11-09T17:28:12,237 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:12,237 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(2837): Flushing d57ffb67cf090d609649a6e4d95584ca 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T17:28:12,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=A 2024-11-09T17:28:12,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:12,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=B 2024-11-09T17:28:12,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:12,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d57ffb67cf090d609649a6e4d95584ca, store=C 2024-11-09T17:28:12,239 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:12,246 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a43d30030724c38a86f5799c98527db is 50, key is test_row_0/A:col10/1731173290110/Put/seqid=0 2024-11-09T17:28:12,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742286_1462 (size=12301) 2024-11-09T17:28:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:12,654 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a43d30030724c38a86f5799c98527db 2024-11-09T17:28:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:12,667 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b233f1c8493b416e9c7ca5d721a6d555 is 50, key is test_row_0/B:col10/1731173290110/Put/seqid=0 2024-11-09T17:28:12,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742287_1463 (size=12301) 2024-11-09T17:28:13,074 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b233f1c8493b416e9c7ca5d721a6d555 2024-11-09T17:28:13,118 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 is 50, key is test_row_0/C:col10/1731173290110/Put/seqid=0 2024-11-09T17:28:13,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742288_1464 (size=12301) 2024-11-09T17:28:13,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:13,190 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:13,525 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 2024-11-09T17:28:13,534 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/A/0a43d30030724c38a86f5799c98527db as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a43d30030724c38a86f5799c98527db 2024-11-09T17:28:13,537 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a43d30030724c38a86f5799c98527db, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T17:28:13,537 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/B/b233f1c8493b416e9c7ca5d721a6d555 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b233f1c8493b416e9c7ca5d721a6d555 2024-11-09T17:28:13,540 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b233f1c8493b416e9c7ca5d721a6d555, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T17:28:13,540 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/.tmp/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 2024-11-09T17:28:13,543 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2f42017f1d4a4ea0aae8e1a14e3e3e27, entries=150, sequenceid=332, filesize=12.0 K 2024-11-09T17:28:13,543 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d57ffb67cf090d609649a6e4d95584ca in 1306ms, sequenceid=332, compaction requested=true 2024-11-09T17:28:13,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/75358ddcbf7245efaee5752d4dcf3b56, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/198493912cda4e489447403499057a5f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4a294a1e71c64a88948a8e6bd88bd6fe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4728a90091ae49489c63edbd8de85184, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1426ecdea633430f850dddb61107ac29, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394] to archive 2024-11-09T17:28:13,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:13,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/f8823d2d6fc748cabdd17d02a7bb0ede 2024-11-09T17:28:13,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/49554d7dc33d4957a882009c5e8118e6 2024-11-09T17:28:13,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1d807f68a98847f4b40c4f563a07ca42 2024-11-09T17:28:13,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/75358ddcbf7245efaee5752d4dcf3b56 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/75358ddcbf7245efaee5752d4dcf3b56 2024-11-09T17:28:13,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a039a8b9fb74164a24d0347de7a45af 2024-11-09T17:28:13,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/712f005833354b65810f84a8b44e85f4 2024-11-09T17:28:13,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/5d38c5a1ea774733946a2c5c1db0f527 2024-11-09T17:28:13,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/198493912cda4e489447403499057a5f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/198493912cda4e489447403499057a5f 2024-11-09T17:28:13,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4ec45457c6fb4602b815c40d78153acf 2024-11-09T17:28:13,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/220e4928f329403c9a8b5e2782a0b135 2024-11-09T17:28:13,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4a294a1e71c64a88948a8e6bd88bd6fe to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4a294a1e71c64a88948a8e6bd88bd6fe 2024-11-09T17:28:13,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/ddcfa3ad03284fdbb60dbdfd4c026dc1 2024-11-09T17:28:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/6eaab712e1e6440b93faefa99526ef07 2024-11-09T17:28:13,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/3ad8c749422c4181b0b62b9836e4ae6e 2024-11-09T17:28:13,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4728a90091ae49489c63edbd8de85184 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/4728a90091ae49489c63edbd8de85184 2024-11-09T17:28:13,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/aacf345df2aa457c8ea6dcef98b7ad72 2024-11-09T17:28:13,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/332b16ab72344c339dbf2bb6f1bac8fa 2024-11-09T17:28:13,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1426ecdea633430f850dddb61107ac29 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1426ecdea633430f850dddb61107ac29 2024-11-09T17:28:13,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/533160cf15404241ab8c894fdbd41872 2024-11-09T17:28:13,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/dc51ea18a6b542d3aa9281610a8826af 2024-11-09T17:28:13,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/9ca9c4da0b7e4e3dbdf781d577c52394 2024-11-09T17:28:13,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/828bf4622a82470985db97cb531fa554, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/183b88db1241432497ab2b0e7d92a2c8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/2b631356810646598c967fd9dcb9c586, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ab05546196f94fb7a17fee1eb2dcc036, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ea0474ba59794a4e9bb6c034ddc6f0c7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0] to archive 2024-11-09T17:28:13,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:13,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/cf9fdbb3148f4fb99798287cc6800041 2024-11-09T17:28:13,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/58b105c1677f448496155e6fd13f89b0 2024-11-09T17:28:13,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/828bf4622a82470985db97cb531fa554 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/828bf4622a82470985db97cb531fa554 2024-11-09T17:28:13,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4040982793af497a801786b1fa765d8b 2024-11-09T17:28:13,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/df49250953cd458785f2f2d2c2cdf5f4 2024-11-09T17:28:13,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/11b0d2b6098045f7b25c31bc026f16ce 2024-11-09T17:28:13,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/183b88db1241432497ab2b0e7d92a2c8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/183b88db1241432497ab2b0e7d92a2c8 2024-11-09T17:28:13,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b9b929c18a2c4ae38c2c9ecd44425428 2024-11-09T17:28:13,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/f5f680bbf2314ea8a18b425bee18f0ed 2024-11-09T17:28:13,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/2b631356810646598c967fd9dcb9c586 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/2b631356810646598c967fd9dcb9c586 2024-11-09T17:28:13,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/d73ba37bf6844b8a976362167d08036e 2024-11-09T17:28:13,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/594f4d86b07b4bbb8ae8316a6ddb8dfb 2024-11-09T17:28:13,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/fd8c7838d43c4c1cb89f2be7d3d8d2ea 2024-11-09T17:28:13,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ab05546196f94fb7a17fee1eb2dcc036 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ab05546196f94fb7a17fee1eb2dcc036 2024-11-09T17:28:13,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/c08c8b4e95834c07a68f421c9e340377 2024-11-09T17:28:13,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/886110fba0b44604b5e3c0557938dfcf 2024-11-09T17:28:13,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ea0474ba59794a4e9bb6c034ddc6f0c7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/ea0474ba59794a4e9bb6c034ddc6f0c7 2024-11-09T17:28:13,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/5381327495914db2ba9fc30b2bd4cf56 2024-11-09T17:28:13,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/94869c0bfb1746e5a938493f6d680a2f 2024-11-09T17:28:13,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/777c671f1931462a8de22f346764bd0e 2024-11-09T17:28:13,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/65d3e24845a84d0f98d2236ecb68ffe0 2024-11-09T17:28:13,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/eb2e55b82ebb4134abba884c1e12b095, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/ddf9ef5bdd4a45a6a2ad1470663c35e1, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e1754269653454bb23d4e1f8c6e4f4d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/44193fe7fc974b448feb7f0e3dfc1d2e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/f423a8a771b74db2bd84a38b55197595, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923] to archive 2024-11-09T17:28:13,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:13,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/dabdcccba2b84a76abee03417195ee2d 2024-11-09T17:28:13,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a27fde54afd8498d94a8d247e6737cfe 2024-11-09T17:28:13,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/eb2e55b82ebb4134abba884c1e12b095 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/eb2e55b82ebb4134abba884c1e12b095 2024-11-09T17:28:13,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/53442d696f5b4ee2965b2a46c015bde3 2024-11-09T17:28:13,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/342dfdf9091c45469603cd95c0781cf1 2024-11-09T17:28:13,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/de1cb803019a4d85959339bf854b91fe 2024-11-09T17:28:13,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/ddf9ef5bdd4a45a6a2ad1470663c35e1 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/ddf9ef5bdd4a45a6a2ad1470663c35e1 2024-11-09T17:28:13,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e21780421ef41198c0a34bac0d8a9bf 2024-11-09T17:28:13,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/626e13cd1b4842f3bcb03f6e36c45dde 2024-11-09T17:28:13,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e1754269653454bb23d4e1f8c6e4f4d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/9e1754269653454bb23d4e1f8c6e4f4d 2024-11-09T17:28:13,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/360065a0b1b845128c7870581acad11e 2024-11-09T17:28:13,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/4232b2f9716e424e992132f665f4f4b0 2024-11-09T17:28:13,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/df496940bd774fabb53acce968c71020 2024-11-09T17:28:13,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/44193fe7fc974b448feb7f0e3dfc1d2e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/44193fe7fc974b448feb7f0e3dfc1d2e 2024-11-09T17:28:13,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/acff3df39d5a415c8fe400b1335e045a 2024-11-09T17:28:13,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/bd319f07afb04b3dba4462b631adf750 2024-11-09T17:28:13,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/f423a8a771b74db2bd84a38b55197595 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/f423a8a771b74db2bd84a38b55197595 2024-11-09T17:28:13,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/3598da3205aa470b91d1596cfd71e193 2024-11-09T17:28:13,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/106a943be05c4aef9a2044d2d0897deb 2024-11-09T17:28:13,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/a16b60581ff94e91995353f21bc90af3 2024-11-09T17:28:13,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2d2c10f43f0a44699fd2ea8d1b6bb923 2024-11-09T17:28:13,604 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=1 2024-11-09T17:28:13,605 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca. 2024-11-09T17:28:13,605 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] regionserver.HRegion(1635): Region close journal for d57ffb67cf090d609649a6e4d95584ca: 2024-11-09T17:28:13,606 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=137}] handler.UnassignRegionHandler(170): Closed d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:13,606 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=136 updating hbase:meta row=d57ffb67cf090d609649a6e4d95584ca, regionState=CLOSED 2024-11-09T17:28:13,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=137, resume processing ppid=136 2024-11-09T17:28:13,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, ppid=136, state=SUCCESS; CloseRegionProcedure d57ffb67cf090d609649a6e4d95584ca, server=c28fd47d982f,45633,1731173138129 in 1.5240 sec 2024-11-09T17:28:13,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-11-09T17:28:13,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=d57ffb67cf090d609649a6e4d95584ca, UNASSIGN in 1.5280 sec 2024-11-09T17:28:13,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=135, resume processing ppid=134 2024-11-09T17:28:13,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, ppid=134, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5300 sec 2024-11-09T17:28:13,611 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173293611"}]},"ts":"1731173293611"} 2024-11-09T17:28:13,611 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:28:13,653 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:28:13,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.5980 sec 2024-11-09T17:28:14,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=134 2024-11-09T17:28:14,168 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 134 completed 2024-11-09T17:28:14,169 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:28:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=138, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,173 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=138, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,175 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=138, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-09T17:28:14,180 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:14,184 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/recovered.edits] 2024-11-09T17:28:14,186 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a43d30030724c38a86f5799c98527db to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/0a43d30030724c38a86f5799c98527db 2024-11-09T17:28:14,187 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1e513fc452ac49f783d30c0e9548ae95 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/1e513fc452ac49f783d30c0e9548ae95 2024-11-09T17:28:14,188 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/98cecaae41254e83a673f33cde593114 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/A/98cecaae41254e83a673f33cde593114 2024-11-09T17:28:14,190 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/1fda9e4d1c16438cb968521852f3ca97 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/1fda9e4d1c16438cb968521852f3ca97 2024-11-09T17:28:14,191 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4ffeda699f3e4e57897992fac24f0726 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/4ffeda699f3e4e57897992fac24f0726 2024-11-09T17:28:14,192 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b233f1c8493b416e9c7ca5d721a6d555 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/B/b233f1c8493b416e9c7ca5d721a6d555 2024-11-09T17:28:14,193 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/18bb699ed22c4773943a2fd3aa968072 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/18bb699ed22c4773943a2fd3aa968072 2024-11-09T17:28:14,194 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/2f42017f1d4a4ea0aae8e1a14e3e3e27 2024-11-09T17:28:14,196 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/5da05a064a594efd86d120d57dfc920f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/C/5da05a064a594efd86d120d57dfc920f 2024-11-09T17:28:14,198 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/recovered.edits/335.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca/recovered.edits/335.seqid 2024-11-09T17:28:14,199 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/d57ffb67cf090d609649a6e4d95584ca 2024-11-09T17:28:14,199 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:28:14,201 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=138, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,203 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:28:14,205 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:28:14,206 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=138, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,206 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:28:14,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173294206"}]},"ts":"9223372036854775807"} 2024-11-09T17:28:14,208 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:28:14,208 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => d57ffb67cf090d609649a6e4d95584ca, NAME => 'TestAcidGuarantees,,1731173265237.d57ffb67cf090d609649a6e4d95584ca.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:28:14,208 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:28:14,209 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173294208"}]},"ts":"9223372036854775807"} 2024-11-09T17:28:14,211 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:28:14,255 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=138, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 85 msec 2024-11-09T17:28:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=138 2024-11-09T17:28:14,277 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 138 completed 2024-11-09T17:28:14,285 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testScanAtomicity Thread=239 (was 236) - Thread LEAK? -, OpenFileDescriptor=451 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=295 (was 303), ProcessCount=11 (was 11), AvailableMemoryMB=5825 (was 5834) 2024-11-09T17:28:14,294 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithBasicPolicy#testMobGetAtomicity Thread=239, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=295, ProcessCount=11, AvailableMemoryMB=5825 2024-11-09T17:28:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:28:14,296 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$4(2389): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:28:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:14,298 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T17:28:14,298 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:14,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(713): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 139 2024-11-09T17:28:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:14,299 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T17:28:14,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742289_1465 (size=960) 2024-11-09T17:28:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:14,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:14,712 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7 2024-11-09T17:28:14,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742290_1466 (size=53) 2024-11-09T17:28:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:15,123 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:28:15,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1681): Closing e267c9faf6557bc65334d4401871cb37, disabling compactions & flushes 2024-11-09T17:28:15,124 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. after waiting 0 ms 2024-11-09T17:28:15,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,124 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,124 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1635): Region close journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:15,127 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T17:28:15,127 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731173295127"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731173295127"}]},"ts":"1731173295127"} 2024-11-09T17:28:15,130 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-09T17:28:15,131 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T17:28:15,132 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173295131"}]},"ts":"1731173295131"} 2024-11-09T17:28:15,133 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T17:28:15,246 ERROR [LeaseRenewer:jenkins@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46491,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:15,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, ASSIGN}] 2024-11-09T17:28:15,290 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, ASSIGN 2024-11-09T17:28:15,292 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=140, ppid=139, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, ASSIGN; state=OFFLINE, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=false 2024-11-09T17:28:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:15,443 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:15,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE; OpenRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:28:15,599 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:15,606 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,606 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7285): Opening region: {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:28:15,607 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,607 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:28:15,607 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7327): checking encryption for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,607 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7330): checking classloading for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,610 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,611 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:15,612 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName A 2024-11-09T17:28:15,612 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:15,612 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:15,612 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,613 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:15,613 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName B 2024-11-09T17:28:15,613 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:15,614 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:15,614 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,614 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:15,615 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName C 2024-11-09T17:28:15,615 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:15,615 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:15,615 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,616 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,616 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,617 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:28:15,618 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1085): writing seq id for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:15,620 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T17:28:15,620 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1102): Opened e267c9faf6557bc65334d4401871cb37; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73037046, jitterRate=0.08833679556846619}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:28:15,620 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1001): Region open journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:15,621 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., pid=141, masterSystemTime=1731173295599 2024-11-09T17:28:15,622 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,622 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:15,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=140 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=OPEN, openSeqNum=2, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:15,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=141, resume processing ppid=140 2024-11-09T17:28:15,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, ppid=140, state=SUCCESS; OpenRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 in 179 msec 2024-11-09T17:28:15,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T17:28:15,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, ASSIGN in 337 msec 2024-11-09T17:28:15,628 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T17:28:15,628 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173295628"}]},"ts":"1731173295628"} 2024-11-09T17:28:15,629 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T17:28:15,688 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=139, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T17:28:15,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; CreateTableProcedure table=TestAcidGuarantees in 1.3920 sec 2024-11-09T17:28:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-09T17:28:16,409 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: CREATE, Table Name: default:TestAcidGuarantees, procId: 139 completed 2024-11-09T17:28:16,412 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06094c70 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5bc9c3e 2024-11-09T17:28:16,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc332d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:16,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:16,481 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60344, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:16,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T17:28:16,484 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42310, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T17:28:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T17:28:16,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$14(2798): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'BASIC', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T17:28:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=142, state=RUNNABLE:MODIFY_TABLE_PREPARE; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:16,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742291_1467 (size=996) 2024-11-09T17:28:16,904 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(519): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.960 2024-11-09T17:28:16,904 INFO [PEWorker-4 {}] util.FSTableDescriptors(297): Updated tableinfo=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.996 2024-11-09T17:28:16,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:28:16,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, REOPEN/MOVE}] 2024-11-09T17:28:16,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, REOPEN/MOVE 2024-11-09T17:28:16,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:16,914 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:28:16,914 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE; CloseRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:28:17,066 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,067 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(124): Close e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,067 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:28:17,067 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1681): Closing e267c9faf6557bc65334d4401871cb37, disabling compactions & flushes 2024-11-09T17:28:17,067 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,067 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,067 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. after waiting 0 ms 2024-11-09T17:28:17,067 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,076 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T17:28:17,077 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,077 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegion(1635): Region close journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:17,077 WARN [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] regionserver.HRegionServer(3786): Not adding moved region record: e267c9faf6557bc65334d4401871cb37 to self. 2024-11-09T17:28:17,079 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=145}] handler.UnassignRegionHandler(170): Closed e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,079 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=CLOSED 2024-11-09T17:28:17,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=145, resume processing ppid=144 2024-11-09T17:28:17,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, ppid=144, state=SUCCESS; CloseRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 in 167 msec 2024-11-09T17:28:17,083 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=144, ppid=143, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, REOPEN/MOVE; state=CLOSED, location=c28fd47d982f,45633,1731173138129; forceNewPlan=false, retain=true 2024-11-09T17:28:17,234 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=OPENING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,237 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=144, state=RUNNABLE; OpenRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:28:17,391 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,397 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(135): Open TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,397 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7285): Opening region: {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} 2024-11-09T17:28:17,398 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,398 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(894): Instantiated TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T17:28:17,398 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7327): checking encryption for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,398 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7330): checking classloading for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,400 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,400 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:17,401 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName A 2024-11-09T17:28:17,402 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:17,403 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:17,403 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,404 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:17,404 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName B 2024-11-09T17:28:17,404 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:17,405 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:17,405 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,406 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=2.00 MB, immutable segments index type=CHUNK_MAP, compactor=BASIC, pipelineThreshold=2, compactionCellMax=10 2024-11-09T17:28:17,406 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e267c9faf6557bc65334d4401871cb37 columnFamilyName C 2024-11-09T17:28:17,406 DEBUG [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:17,407 INFO [StoreOpener-e267c9faf6557bc65334d4401871cb37-1 {}] regionserver.HStore(327): Store=e267c9faf6557bc65334d4401871cb37/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T17:28:17,407 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1178): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,408 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,409 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,411 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T17:28:17,414 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1085): writing seq id for e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,415 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1102): Opened e267c9faf6557bc65334d4401871cb37; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68023669, jitterRate=0.013631656765937805}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T17:28:17,415 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1001): Region open journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:17,416 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2601): Post open deploy tasks for TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., pid=146, masterSystemTime=1731173297390 2024-11-09T17:28:17,417 DEBUG [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2628): Finished post open deploy task for TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,417 INFO [RS_OPEN_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(164): Opened TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,417 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=144 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=OPEN, openSeqNum=5, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=144 2024-11-09T17:28:17,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=144, state=SUCCESS; OpenRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 in 181 msec 2024-11-09T17:28:17,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-11-09T17:28:17,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, REOPEN/MOVE in 507 msec 2024-11-09T17:28:17,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=143, resume processing ppid=142 2024-11-09T17:28:17,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, ppid=142, state=SUCCESS; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-09T17:28:17,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, state=SUCCESS; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-09T17:28:17,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=142 2024-11-09T17:28:17,424 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x103dfc6e to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7181df3b 2024-11-09T17:28:17,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17327621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,472 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e047c09 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11030ef5 2024-11-09T17:28:17,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1584f18a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,486 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60d631a3 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@69abefea 2024-11-09T17:28:17,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b914bf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,496 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58971172 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6e757135 2024-11-09T17:28:17,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6a59e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,504 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d7fe93b to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7846cb78 2024-11-09T17:28:17,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150e08ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,512 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f1754bc to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a3b66d3 2024-11-09T17:28:17,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6288a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,521 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d9113f3 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cfdf76c 2024-11-09T17:28:17,529 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6556601, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,529 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5bb75907 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68c2838a 2024-11-09T17:28:17,537 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@458a85fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,537 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c1d3a95 to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@50bf224f 2024-11-09T17:28:17,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@410bf0c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,545 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x503a7d2e to 127.0.0.1:57222 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79be903c 2024-11-09T17:28:17,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67adb273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T17:28:17,556 DEBUG [hconnection-0x4e006260-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,556 DEBUG [hconnection-0x649dc7c7-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:17,557 DEBUG [hconnection-0x32e84a7c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,558 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60358, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,558 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60356, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,558 DEBUG [hconnection-0x6128a2f-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-09T17:28:17,559 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60362, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,560 DEBUG [hconnection-0x7727a6c5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,560 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60376, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:17,561 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60378, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,561 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:17,561 DEBUG [hconnection-0x6ff04539-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,561 DEBUG [hconnection-0x13c2a26c-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,561 DEBUG [hconnection-0x2d1f74d5-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,562 DEBUG [hconnection-0x2e1c2ea0-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,562 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60390, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,562 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:17,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:17,563 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60404, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,563 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60398, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,563 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60414, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,566 DEBUG [hconnection-0x7ba8a9e-metaLookup-shared--pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T17:28:17,567 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60424, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T17:28:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:17,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:17,569 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:17,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173357579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173357579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173357580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173357580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173357581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095dd1686d372d4ff984ae84f0d7834991_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173297567/Put/seqid=0 2024-11-09T17:28:17,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742292_1468 (size=12154) 2024-11-09T17:28:17,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173357682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173357682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173357683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173357683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173357683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,715 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:17,867 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:17,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173357884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173357885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173357885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173357885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:17,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173357885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,000 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:18,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095dd1686d372d4ff984ae84f0d7834991_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095dd1686d372d4ff984ae84f0d7834991_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:18,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1e5ea8bfd3ab49ae891532487eb070d2, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:18,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1e5ea8bfd3ab49ae891532487eb070d2 is 175, key is test_row_0/A:col10/1731173297567/Put/seqid=0 2024-11-09T17:28:18,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742293_1469 (size=30955) 2024-11-09T17:28:18,020 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:18,172 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173358189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,324 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,409 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1e5ea8bfd3ab49ae891532487eb070d2 2024-11-09T17:28:18,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/200aeba9aa0b448d87a53899347b6864 is 50, key is test_row_0/B:col10/1731173297567/Put/seqid=0 2024-11-09T17:28:18,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742294_1470 (size=12001) 2024-11-09T17:28:18,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/200aeba9aa0b448d87a53899347b6864 2024-11-09T17:28:18,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/234bd444aafc4dc18a804fc580772887 is 50, key is test_row_0/C:col10/1731173297567/Put/seqid=0 2024-11-09T17:28:18,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742295_1471 (size=12001) 2024-11-09T17:28:18,476 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,628 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173358690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173358691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173358694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:18,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173358695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,781 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:18,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/234bd444aafc4dc18a804fc580772887 2024-11-09T17:28:18,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1e5ea8bfd3ab49ae891532487eb070d2 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2 2024-11-09T17:28:18,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2, entries=150, sequenceid=17, filesize=30.2 K 2024-11-09T17:28:18,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/200aeba9aa0b448d87a53899347b6864 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864 2024-11-09T17:28:18,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:28:18,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/234bd444aafc4dc18a804fc580772887 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887 2024-11-09T17:28:18,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887, entries=150, sequenceid=17, filesize=11.7 K 2024-11-09T17:28:18,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for e267c9faf6557bc65334d4401871cb37 in 1293ms, sequenceid=17, compaction requested=false 2024-11-09T17:28:18,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:18,938 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:18,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T17:28:18,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:18,939 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-09T17:28:18,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:18,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:18,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:18,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:18,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:18,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:18,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110989a277aee605449d9366d423519ce299_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173297579/Put/seqid=0 2024-11-09T17:28:18,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742296_1472 (size=12154) 2024-11-09T17:28:18,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:18,960 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110989a277aee605449d9366d423519ce299_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989a277aee605449d9366d423519ce299_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:18,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/11afc75b8caf4acdb644277ec55f66b5, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:18,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/11afc75b8caf4acdb644277ec55f66b5 is 175, key is test_row_0/A:col10/1731173297579/Put/seqid=0 2024-11-09T17:28:18,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742297_1473 (size=30955) 2024-11-09T17:28:19,130 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T17:28:19,364 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/11afc75b8caf4acdb644277ec55f66b5 2024-11-09T17:28:19,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7d5e02fcd9d447678abb96963a740bd6 is 50, key is test_row_0/B:col10/1731173297579/Put/seqid=0 2024-11-09T17:28:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742298_1474 (size=12001) 2024-11-09T17:28:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:19,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173359701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173359701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173359702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173359704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173359705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,776 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7d5e02fcd9d447678abb96963a740bd6 2024-11-09T17:28:19,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2b9d65901e214ff7a8a7c5be453f8585 is 50, key is test_row_0/C:col10/1731173297579/Put/seqid=0 2024-11-09T17:28:19,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742299_1475 (size=12001) 2024-11-09T17:28:19,785 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2b9d65901e214ff7a8a7c5be453f8585 2024-11-09T17:28:19,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/11afc75b8caf4acdb644277ec55f66b5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5 2024-11-09T17:28:19,793 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-09T17:28:19,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7d5e02fcd9d447678abb96963a740bd6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6 2024-11-09T17:28:19,796 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T17:28:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2b9d65901e214ff7a8a7c5be453f8585 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585 2024-11-09T17:28:19,800 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585, entries=150, sequenceid=40, filesize=11.7 K 2024-11-09T17:28:19,801 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3040): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for e267c9faf6557bc65334d4401871cb37 in 862ms, sequenceid=40, compaction requested=false 2024-11-09T17:28:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-09T17:28:19,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=148 2024-11-09T17:28:19,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=148, resume processing ppid=147 2024-11-09T17:28:19,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=148, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-09T17:28:19,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:19,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-09T17:28:19,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 2.2500 sec 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:19,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:19,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109086364ae16524401ab79dff098b2a225_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:19,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742300_1476 (size=17034) 2024-11-09T17:28:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173359822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173359854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173359955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173359957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173359957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173359957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173360158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173360159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173360159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173360160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:20,223 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109086364ae16524401ab79dff098b2a225_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109086364ae16524401ab79dff098b2a225_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:20,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/34bcf2e2b5554ff98ab679dc0ec316e8, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:20,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/34bcf2e2b5554ff98ab679dc0ec316e8 is 175, key is test_row_0/A:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:20,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742301_1477 (size=48139) 2024-11-09T17:28:20,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173360462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173360463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173360463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173360464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/34bcf2e2b5554ff98ab679dc0ec316e8 2024-11-09T17:28:20,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c1eaab287d404722a54e8e6a019f5be0 is 50, key is test_row_0/B:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742302_1478 (size=12001) 2024-11-09T17:28:20,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c1eaab287d404722a54e8e6a019f5be0 2024-11-09T17:28:20,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/10b6dcee6f064ec6b2a56c811cbd5d62 is 50, key is test_row_0/C:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742303_1479 (size=12001) 2024-11-09T17:28:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173360966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173360967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173360967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:20,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:20,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173360968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/10b6dcee6f064ec6b2a56c811cbd5d62 2024-11-09T17:28:21,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/34bcf2e2b5554ff98ab679dc0ec316e8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8 2024-11-09T17:28:21,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8, entries=250, sequenceid=56, filesize=47.0 K 2024-11-09T17:28:21,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c1eaab287d404722a54e8e6a019f5be0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0 2024-11-09T17:28:21,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T17:28:21,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/10b6dcee6f064ec6b2a56c811cbd5d62 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62 2024-11-09T17:28:21,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62, entries=150, sequenceid=56, filesize=11.7 K 2024-11-09T17:28:21,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for e267c9faf6557bc65334d4401871cb37 in 1250ms, sequenceid=56, compaction requested=true 2024-11-09T17:28:21,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:21,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:21,056 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:21,056 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:21,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:21,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:21,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:21,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:21,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:21,057 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:21,057 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:21,057 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/B is initiating minor compaction (all files) 2024-11-09T17:28:21,057 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/A is initiating minor compaction (all files) 2024-11-09T17:28:21,057 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/B in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,057 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/A in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,057 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=35.2 K 2024-11-09T17:28:21,057 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=107.5 K 2024-11-09T17:28:21,057 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,057 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8] 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1e5ea8bfd3ab49ae891532487eb070d2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173297566 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 200aeba9aa0b448d87a53899347b6864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173297566 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7d5e02fcd9d447678abb96963a740bd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731173297579 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 11afc75b8caf4acdb644277ec55f66b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731173297579 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c1eaab287d404722a54e8e6a019f5be0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299805 2024-11-09T17:28:21,058 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 34bcf2e2b5554ff98ab679dc0ec316e8, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299701 2024-11-09T17:28:21,062 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:21,064 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110984be9a2e10cb4194afca09067ec1cecf_e267c9faf6557bc65334d4401871cb37 store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:21,064 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#B#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:21,065 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/e204d1cb6f174056b79f637a65f14d8a is 50, key is test_row_0/B:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:21,066 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110984be9a2e10cb4194afca09067ec1cecf_e267c9faf6557bc65334d4401871cb37, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:21,066 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110984be9a2e10cb4194afca09067ec1cecf_e267c9faf6557bc65334d4401871cb37 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:21,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742304_1480 (size=12104) 2024-11-09T17:28:21,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742305_1481 (size=4469) 2024-11-09T17:28:21,072 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/e204d1cb6f174056b79f637a65f14d8a as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/e204d1cb6f174056b79f637a65f14d8a 2024-11-09T17:28:21,076 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/B of e267c9faf6557bc65334d4401871cb37 into e204d1cb6f174056b79f637a65f14d8a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:21,076 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:21,076 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/B, priority=13, startTime=1731173301056; duration=0sec 2024-11-09T17:28:21,076 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:21,076 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:B 2024-11-09T17:28:21,076 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:21,076 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:21,077 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/C is initiating minor compaction (all files) 2024-11-09T17:28:21,077 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/C in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,077 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=35.2 K 2024-11-09T17:28:21,077 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 234bd444aafc4dc18a804fc580772887, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731173297566 2024-11-09T17:28:21,077 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2b9d65901e214ff7a8a7c5be453f8585, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731173297579 2024-11-09T17:28:21,077 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 10b6dcee6f064ec6b2a56c811cbd5d62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299805 2024-11-09T17:28:21,082 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#C#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:21,082 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1ac9306b6e6a4948aa63a03c9c70ec22 is 50, key is test_row_0/C:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:21,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742306_1482 (size=12104) 2024-11-09T17:28:21,470 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#A#compaction#404 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:21,470 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1fbba25eec5f4f48af37c92310538c5e is 175, key is test_row_0/A:col10/1731173299805/Put/seqid=0 2024-11-09T17:28:21,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742307_1483 (size=31058) 2024-11-09T17:28:21,511 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1ac9306b6e6a4948aa63a03c9c70ec22 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1ac9306b6e6a4948aa63a03c9c70ec22 2024-11-09T17:28:21,514 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/C of e267c9faf6557bc65334d4401871cb37 into 1ac9306b6e6a4948aa63a03c9c70ec22(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:21,514 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:21,515 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/C, priority=13, startTime=1731173301057; duration=0sec 2024-11-09T17:28:21,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:21,515 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:C 2024-11-09T17:28:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-09T17:28:21,665 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 147 completed 2024-11-09T17:28:21,666 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees 2024-11-09T17:28:21,667 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:21,668 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=149, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:21,668 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:21,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:21,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:21,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f4b484af4b0d4997b236c340234b3e23_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173299827/Put/seqid=0 2024-11-09T17:28:21,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742308_1484 (size=12154) 2024-11-09T17:28:21,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173361741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:21,819 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173361844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,877 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1fbba25eec5f4f48af37c92310538c5e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e 2024-11-09T17:28:21,880 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/A of e267c9faf6557bc65334d4401871cb37 into 1fbba25eec5f4f48af37c92310538c5e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:21,880 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:21,881 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/A, priority=13, startTime=1731173301056; duration=0sec 2024-11-09T17:28:21,881 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:21,881 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:A 2024-11-09T17:28:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:21,971 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:21,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173361971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:21,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173361973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173361974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:21,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173361977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173362045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,119 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:22,122 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f4b484af4b0d4997b236c340234b3e23_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f4b484af4b0d4997b236c340234b3e23_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:22,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d21f89e6a21b46f291ae4aba0c1a99b8, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:22,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d21f89e6a21b46f291ae4aba0c1a99b8 is 175, key is test_row_0/A:col10/1731173299827/Put/seqid=0 2024-11-09T17:28:22,123 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742309_1485 (size=30955) 2024-11-09T17:28:22,126 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d21f89e6a21b46f291ae4aba0c1a99b8 2024-11-09T17:28:22,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/35da710add2c41a1a5ab7f96ff211611 is 50, key is test_row_0/B:col10/1731173299827/Put/seqid=0 2024-11-09T17:28:22,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742310_1486 (size=12001) 2024-11-09T17:28:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:22,275 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173362349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,427 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/35da710add2c41a1a5ab7f96ff211611 2024-11-09T17:28:22,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1144957abf4c445f850b73bb351e3a12 is 50, key is test_row_0/C:col10/1731173299827/Put/seqid=0 2024-11-09T17:28:22,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742311_1487 (size=12001) 2024-11-09T17:28:22,580 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,580 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,732 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:22,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:22,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173362854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,883 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:22,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:22,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] handler.RSProcedureHandler(58): pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=150 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=150 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:22,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1144957abf4c445f850b73bb351e3a12 2024-11-09T17:28:22,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d21f89e6a21b46f291ae4aba0c1a99b8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8 2024-11-09T17:28:22,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8, entries=150, sequenceid=79, filesize=30.2 K 2024-11-09T17:28:22,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/35da710add2c41a1a5ab7f96ff211611 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611 2024-11-09T17:28:22,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T17:28:22,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1144957abf4c445f850b73bb351e3a12 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12 2024-11-09T17:28:22,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12, entries=150, sequenceid=79, filesize=11.7 K 2024-11-09T17:28:22,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e267c9faf6557bc65334d4401871cb37 in 1249ms, sequenceid=79, compaction requested=false 2024-11-09T17:28:22,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:23,036 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=150 2024-11-09T17:28:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:23,037 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090702cbfe03684eef9dc8b1b8950b18da_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173301740/Put/seqid=0 2024-11-09T17:28:23,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742312_1488 (size=12154) 2024-11-09T17:28:23,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:23,452 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090702cbfe03684eef9dc8b1b8950b18da_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090702cbfe03684eef9dc8b1b8950b18da_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/207552f0f52f4367818b2746e7d96289, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/207552f0f52f4367818b2746e7d96289 is 175, key is test_row_0/A:col10/1731173301740/Put/seqid=0 2024-11-09T17:28:23,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742313_1489 (size=30955) 2024-11-09T17:28:23,456 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/207552f0f52f4367818b2746e7d96289 2024-11-09T17:28:23,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ebc01bb6a1a248ea8208e9e0651c2899 is 50, key is test_row_0/B:col10/1731173301740/Put/seqid=0 2024-11-09T17:28:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742314_1490 (size=12001) 2024-11-09T17:28:23,465 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ebc01bb6a1a248ea8208e9e0651c2899 2024-11-09T17:28:23,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/7d32d7951bd94c1fa7357f25d10ce2a3 is 50, key is test_row_0/C:col10/1731173301740/Put/seqid=0 2024-11-09T17:28:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742315_1491 (size=12001) 2024-11-09T17:28:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:23,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:23,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:23,873 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/7d32d7951bd94c1fa7357f25d10ce2a3 2024-11-09T17:28:23,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/207552f0f52f4367818b2746e7d96289 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289 2024-11-09T17:28:23,879 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289, entries=150, sequenceid=95, filesize=30.2 K 2024-11-09T17:28:23,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ebc01bb6a1a248ea8208e9e0651c2899 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899 2024-11-09T17:28:23,882 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T17:28:23,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/7d32d7951bd94c1fa7357f25d10ce2a3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3 2024-11-09T17:28:23,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T17:28:23,886 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(3040): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=46.96 KB/48090 for e267c9faf6557bc65334d4401871cb37 in 850ms, sequenceid=95, compaction requested=true 2024-11-09T17:28:23,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:23,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:23,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-11-09T17:28:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=150 2024-11-09T17:28:23,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=150, resume processing ppid=149 2024-11-09T17:28:23,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=150, ppid=149, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2190 sec 2024-11-09T17:28:23,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:23,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:23,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:23,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=149, table=TestAcidGuarantees in 2.2220 sec 2024-11-09T17:28:23,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cc642dc5a36a45d7966aabfb0977e518_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:23,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742316_1492 (size=14594) 2024-11-09T17:28:23,898 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:23,901 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109cc642dc5a36a45d7966aabfb0977e518_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cc642dc5a36a45d7966aabfb0977e518_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:23,902 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1132ea16c7c74cba92521e9d4f9a82bf, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:23,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1132ea16c7c74cba92521e9d4f9a82bf is 175, key is test_row_0/A:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:23,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742317_1493 (size=39549) 2024-11-09T17:28:23,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1132ea16c7c74cba92521e9d4f9a82bf 2024-11-09T17:28:23,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7ca8b7cd2046440889bb991281395617 is 50, key is test_row_0/B:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:23,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742318_1494 (size=12001) 2024-11-09T17:28:23,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173363967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173363981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,982 DEBUG [Thread-2118 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4128 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:23,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173363982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,984 DEBUG [Thread-2114 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4162 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:23,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:23,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173363985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,987 DEBUG [Thread-2116 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4133 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:23,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173363996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:23,997 DEBUG [Thread-2112 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=6, retries=16, started=4143 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:24,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:24,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173364071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:24,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:24,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173364274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:24,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7ca8b7cd2046440889bb991281395617 2024-11-09T17:28:24,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/ace2a76ba58c45a7afbf06b8d07ce778 is 50, key is test_row_0/C:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:24,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742319_1495 (size=12001) 2024-11-09T17:28:24,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173364577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:24,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/ace2a76ba58c45a7afbf06b8d07ce778 2024-11-09T17:28:24,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/1132ea16c7c74cba92521e9d4f9a82bf as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf 2024-11-09T17:28:24,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf, entries=200, sequenceid=106, filesize=38.6 K 2024-11-09T17:28:24,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/7ca8b7cd2046440889bb991281395617 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617 2024-11-09T17:28:24,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617, entries=150, sequenceid=106, filesize=11.7 K 2024-11-09T17:28:24,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/ace2a76ba58c45a7afbf06b8d07ce778 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778 2024-11-09T17:28:24,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778, entries=150, sequenceid=106, filesize=11.7 K 2024-11-09T17:28:24,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e267c9faf6557bc65334d4401871cb37 in 859ms, sequenceid=106, compaction requested=true 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:24,748 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:24,748 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:24,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:24,749 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:24,749 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:24,749 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/B is initiating minor compaction (all files) 2024-11-09T17:28:24,749 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/A is initiating minor compaction (all files) 2024-11-09T17:28:24,749 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/B in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:24,749 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/A in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:24,749 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=129.4 K 2024-11-09T17:28:24,749 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/e204d1cb6f174056b79f637a65f14d8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.0 K 2024-11-09T17:28:24,749 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:24,749 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf] 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting e204d1cb6f174056b79f637a65f14d8a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299805 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1fbba25eec5f4f48af37c92310538c5e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299805 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 35da710add2c41a1a5ab7f96ff211611, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173299817 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d21f89e6a21b46f291ae4aba0c1a99b8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173299817 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ebc01bb6a1a248ea8208e9e0651c2899, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731173301714 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 207552f0f52f4367818b2746e7d96289, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731173301714 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7ca8b7cd2046440889bb991281395617, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:24,750 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 1132ea16c7c74cba92521e9d4f9a82bf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:24,759 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:24,759 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/af5d30135d47414fbbe9983188f519f5 is 50, key is test_row_0/B:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:24,760 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:24,763 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109f963fb2e5c6a404e87c4da768fcfe528_e267c9faf6557bc65334d4401871cb37 store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:24,766 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109f963fb2e5c6a404e87c4da768fcfe528_e267c9faf6557bc65334d4401871cb37, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:24,766 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109f963fb2e5c6a404e87c4da768fcfe528_e267c9faf6557bc65334d4401871cb37 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:24,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742320_1496 (size=12241) 2024-11-09T17:28:24,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742321_1497 (size=4469) 2024-11-09T17:28:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:25,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:25,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:25,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a4e78f3477644974b08ea792c0449fd5_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173305083/Put/seqid=0 2024-11-09T17:28:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742322_1498 (size=14694) 2024-11-09T17:28:25,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:25,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173365106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:25,171 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/af5d30135d47414fbbe9983188f519f5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/af5d30135d47414fbbe9983188f519f5 2024-11-09T17:28:25,173 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#A#compaction#417 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:25,174 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/0e4c8c8c5be74265b942cae49001b9be is 175, key is test_row_0/A:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:25,175 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/B of e267c9faf6557bc65334d4401871cb37 into af5d30135d47414fbbe9983188f519f5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:25,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:25,176 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/B, priority=12, startTime=1731173304748; duration=0sec 2024-11-09T17:28:25,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:25,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:B 2024-11-09T17:28:25,176 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:25,177 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:25,177 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/C is initiating minor compaction (all files) 2024-11-09T17:28:25,177 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/C in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:25,177 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1ac9306b6e6a4948aa63a03c9c70ec22, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.0 K 2024-11-09T17:28:25,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742323_1499 (size=31195) 2024-11-09T17:28:25,177 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1ac9306b6e6a4948aa63a03c9c70ec22, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731173299805 2024-11-09T17:28:25,178 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 1144957abf4c445f850b73bb351e3a12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731173299817 2024-11-09T17:28:25,178 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 7d32d7951bd94c1fa7357f25d10ce2a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731173301714 2024-11-09T17:28:25,178 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ace2a76ba58c45a7afbf06b8d07ce778, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:25,185 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:25,185 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/0129117af7c34ab9adcc71c29cde90c7 is 50, key is test_row_0/C:col10/1731173303888/Put/seqid=0 2024-11-09T17:28:25,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742324_1500 (size=12241) 2024-11-09T17:28:25,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173365209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:25,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173365412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:25,498 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:25,500 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a4e78f3477644974b08ea792c0449fd5_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4e78f3477644974b08ea792c0449fd5_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:25,501 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/cb324b8bf36f45f581dd4fef587fd67d, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:25,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/cb324b8bf36f45f581dd4fef587fd67d is 175, key is test_row_0/A:col10/1731173305083/Put/seqid=0 2024-11-09T17:28:25,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742325_1501 (size=39649) 2024-11-09T17:28:25,581 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/0e4c8c8c5be74265b942cae49001b9be as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be 2024-11-09T17:28:25,584 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/A of e267c9faf6557bc65334d4401871cb37 into 0e4c8c8c5be74265b942cae49001b9be(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:25,584 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:25,584 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/A, priority=12, startTime=1731173304748; duration=0sec 2024-11-09T17:28:25,584 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:25,584 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:A 2024-11-09T17:28:25,591 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/0129117af7c34ab9adcc71c29cde90c7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/0129117af7c34ab9adcc71c29cde90c7 2024-11-09T17:28:25,594 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/C of e267c9faf6557bc65334d4401871cb37 into 0129117af7c34ab9adcc71c29cde90c7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:25,594 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:25,594 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/C, priority=12, startTime=1731173304748; duration=0sec 2024-11-09T17:28:25,594 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:25,594 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:C 2024-11-09T17:28:25,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173365718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=149 2024-11-09T17:28:25,772 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 149 completed 2024-11-09T17:28:25,772 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees 2024-11-09T17:28:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:25,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:25,782 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=151, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:25,782 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:25,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:25,905 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/cb324b8bf36f45f581dd4fef587fd67d 2024-11-09T17:28:25,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/5c04fbfa62e94163821f0349afba5afd is 50, key is test_row_0/B:col10/1731173305083/Put/seqid=0 2024-11-09T17:28:25,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742326_1502 (size=12101) 2024-11-09T17:28:25,934 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:25,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:25,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:25,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:25,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:26,086 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173366221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,238 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/5c04fbfa62e94163821f0349afba5afd 2024-11-09T17:28:26,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/5ca4314d472d414ca1b84a9c4060c8e0 is 50, key is test_row_0/C:col10/1731173305083/Put/seqid=0 2024-11-09T17:28:26,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742327_1503 (size=12101) 2024-11-09T17:28:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:26,390 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,543 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,695 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:26,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] handler.RSProcedureHandler(58): pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=152 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=152 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:26,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/5ca4314d472d414ca1b84a9c4060c8e0 2024-11-09T17:28:26,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/cb324b8bf36f45f581dd4fef587fd67d as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d 2024-11-09T17:28:26,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d, entries=200, sequenceid=132, filesize=38.7 K 2024-11-09T17:28:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/5c04fbfa62e94163821f0349afba5afd as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd 2024-11-09T17:28:26,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T17:28:26,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/5ca4314d472d414ca1b84a9c4060c8e0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0 2024-11-09T17:28:26,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T17:28:26,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for e267c9faf6557bc65334d4401871cb37 in 1656ms, sequenceid=132, compaction requested=false 2024-11-09T17:28:26,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:26,847 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:26,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=152 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:26,848 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:26,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:26,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094f60a45f149544c194321080d2005f99_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173305105/Put/seqid=0 2024-11-09T17:28:26,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742328_1504 (size=12304) 2024-11-09T17:28:26,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:26,860 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094f60a45f149544c194321080d2005f99_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094f60a45f149544c194321080d2005f99_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:26,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/dd6ad85a1f0c471f98686bf8938d98e6, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:26,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/dd6ad85a1f0c471f98686bf8938d98e6 is 175, key is test_row_0/A:col10/1731173305105/Put/seqid=0 2024-11-09T17:28:26,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742329_1505 (size=31105) 2024-11-09T17:28:26,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:27,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:27,266 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/dd6ad85a1f0c471f98686bf8938d98e6 2024-11-09T17:28:27,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ee2b1b33ed0f4e9789935fdc62836a43 is 50, key is test_row_0/B:col10/1731173305105/Put/seqid=0 2024-11-09T17:28:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742330_1506 (size=12151) 2024-11-09T17:28:27,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:27,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173367289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:27,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173367392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173367595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:27,674 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ee2b1b33ed0f4e9789935fdc62836a43 2024-11-09T17:28:27,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8aa266f38b5d4300ad4e963da3cb512f is 50, key is test_row_0/C:col10/1731173305105/Put/seqid=0 2024-11-09T17:28:27,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742331_1507 (size=12151) 2024-11-09T17:28:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:27,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173367897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173368017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,018 DEBUG [Thread-2114 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8196 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:28,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173368020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,022 DEBUG [Thread-2118 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8168 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:28,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173368025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,026 DEBUG [Thread-2112 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8172 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173368027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,028 DEBUG [Thread-2116 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=7, retries=16, started=8174 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:28,090 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8aa266f38b5d4300ad4e963da3cb512f 2024-11-09T17:28:28,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/dd6ad85a1f0c471f98686bf8938d98e6 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6 2024-11-09T17:28:28,095 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6, entries=150, sequenceid=145, filesize=30.4 K 2024-11-09T17:28:28,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/ee2b1b33ed0f4e9789935fdc62836a43 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43 2024-11-09T17:28:28,099 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T17:28:28,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8aa266f38b5d4300ad4e963da3cb512f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f 2024-11-09T17:28:28,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f, entries=150, sequenceid=145, filesize=11.9 K 2024-11-09T17:28:28,102 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(3040): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for e267c9faf6557bc65334d4401871cb37 in 1254ms, sequenceid=145, compaction requested=true 2024-11-09T17:28:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-09T17:28:28,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-11-09T17:28:28,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-11-09T17:28:28,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-09T17:28:28,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=151, table=TestAcidGuarantees in 2.3310 sec 2024-11-09T17:28:28,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:28,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:28,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:28,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e025fc66c58d49b598ed17f59f07febb_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:28,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742332_1508 (size=14794) 2024-11-09T17:28:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173368421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173368524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173368725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:28,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:28,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e025fc66c58d49b598ed17f59f07febb_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e025fc66c58d49b598ed17f59f07febb_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:28,819 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/3398cc56dec0427f92819d8590769dee, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:28,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/3398cc56dec0427f92819d8590769dee is 175, key is test_row_0/A:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:28,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742333_1509 (size=39749) 2024-11-09T17:28:29,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173369030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:29,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/3398cc56dec0427f92819d8590769dee 2024-11-09T17:28:29,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c74f62525974432d807add5cace57276 is 50, key is test_row_0/B:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:29,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742334_1510 (size=12151) 2024-11-09T17:28:29,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173369534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:29,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c74f62525974432d807add5cace57276 2024-11-09T17:28:29,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d2696b24820543cd82fe60f7b06f17c8 is 50, key is test_row_0/C:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:29,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742335_1511 (size=12151) 2024-11-09T17:28:29,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-09T17:28:29,877 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 151 completed 2024-11-09T17:28:29,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees 2024-11-09T17:28:29,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:29,879 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:29,880 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=153, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:29,880 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:29,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:30,031 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:30,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-09T17:28:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] handler.RSProcedureHandler(58): pid=154 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:30,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=154 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=154 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:30,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d2696b24820543cd82fe60f7b06f17c8 2024-11-09T17:28:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/3398cc56dec0427f92819d8590769dee as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee 2024-11-09T17:28:30,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee, entries=200, sequenceid=172, filesize=38.8 K 2024-11-09T17:28:30,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c74f62525974432d807add5cace57276 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276 2024-11-09T17:28:30,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T17:28:30,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d2696b24820543cd82fe60f7b06f17c8 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8 2024-11-09T17:28:30,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T17:28:30,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e267c9faf6557bc65334d4401871cb37 in 1649ms, sequenceid=172, compaction requested=true 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:30,054 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:30,054 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:30,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:30,055 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141698 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:30,055 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:30,055 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/A is initiating minor compaction (all files) 2024-11-09T17:28:30,055 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/B is initiating minor compaction (all files) 2024-11-09T17:28:30,055 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/A in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,055 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/B in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,055 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=138.4 K 2024-11-09T17:28:30,055 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/af5d30135d47414fbbe9983188f519f5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.5 K 2024-11-09T17:28:30,055 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee] 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting af5d30135d47414fbbe9983188f519f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0e4c8c8c5be74265b942cae49001b9be, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 5c04fbfa62e94163821f0349afba5afd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173303950 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting cb324b8bf36f45f581dd4fef587fd67d, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173303950 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting ee2b1b33ed0f4e9789935fdc62836a43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731173305097 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting dd6ad85a1f0c471f98686bf8938d98e6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731173305097 2024-11-09T17:28:30,056 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting c74f62525974432d807add5cace57276, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307282 2024-11-09T17:28:30,057 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3398cc56dec0427f92819d8590769dee, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307266 2024-11-09T17:28:30,061 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:30,063 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411095dbc07fac0b24f3d83b695e3a60d36b5_e267c9faf6557bc65334d4401871cb37 store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:30,063 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:30,064 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/39cb8c09ab234d6b8a7d7ebe5aaf9394 is 50, key is test_row_0/B:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:30,065 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411095dbc07fac0b24f3d83b695e3a60d36b5_e267c9faf6557bc65334d4401871cb37, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:30,065 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095dbc07fac0b24f3d83b695e3a60d36b5_e267c9faf6557bc65334d4401871cb37 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:30,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742337_1513 (size=4469) 2024-11-09T17:28:30,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742336_1512 (size=12527) 2024-11-09T17:28:30,081 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#A#compaction#428 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:30,081 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/891d563360eb45d899b416887d6675e0 is 175, key is test_row_0/A:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:30,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742338_1514 (size=31481) 2024-11-09T17:28:30,088 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/891d563360eb45d899b416887d6675e0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0 2024-11-09T17:28:30,092 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/A of e267c9faf6557bc65334d4401871cb37 into 891d563360eb45d899b416887d6675e0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:30,092 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:30,092 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/A, priority=12, startTime=1731173310054; duration=0sec 2024-11-09T17:28:30,092 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:30,092 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:A 2024-11-09T17:28:30,092 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:30,093 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48644 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:30,093 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/C is initiating minor compaction (all files) 2024-11-09T17:28:30,093 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/C in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,093 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/0129117af7c34ab9adcc71c29cde90c7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.5 K 2024-11-09T17:28:30,093 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 0129117af7c34ab9adcc71c29cde90c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731173303870 2024-11-09T17:28:30,093 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 5ca4314d472d414ca1b84a9c4060c8e0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731173303950 2024-11-09T17:28:30,094 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8aa266f38b5d4300ad4e963da3cb512f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731173305097 2024-11-09T17:28:30,094 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d2696b24820543cd82fe60f7b06f17c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307282 2024-11-09T17:28:30,100 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:30,100 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/6c4ed775b08449889bbdf58a7c2ec383 is 50, key is test_row_0/C:col10/1731173307288/Put/seqid=0 2024-11-09T17:28:30,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742339_1515 (size=12527) 2024-11-09T17:28:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:30,183 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:30,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=154 2024-11-09T17:28:30,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:30,184 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:30,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ed08f25647af451c8606d93da97b9424_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173308415/Put/seqid=0 2024-11-09T17:28:30,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742340_1516 (size=12304) 2024-11-09T17:28:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:30,483 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/39cb8c09ab234d6b8a7d7ebe5aaf9394 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/39cb8c09ab234d6b8a7d7ebe5aaf9394 2024-11-09T17:28:30,497 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/B of e267c9faf6557bc65334d4401871cb37 into 39cb8c09ab234d6b8a7d7ebe5aaf9394(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:30,497 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:30,497 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/B, priority=12, startTime=1731173310054; duration=0sec 2024-11-09T17:28:30,497 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:30,497 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:B 2024-11-09T17:28:30,514 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/6c4ed775b08449889bbdf58a7c2ec383 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/6c4ed775b08449889bbdf58a7c2ec383 2024-11-09T17:28:30,518 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/C of e267c9faf6557bc65334d4401871cb37 into 6c4ed775b08449889bbdf58a7c2ec383(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:30,518 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:30,518 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/C, priority=12, startTime=1731173310054; duration=0sec 2024-11-09T17:28:30,518 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:30,518 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:C 2024-11-09T17:28:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:30,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:30,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:30,596 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ed08f25647af451c8606d93da97b9424_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed08f25647af451c8606d93da97b9424_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:30,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173370595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:30,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/977a877df997451c98f86ef05e440ccb, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:30,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/977a877df997451c98f86ef05e440ccb is 175, key is test_row_0/A:col10/1731173308415/Put/seqid=0 2024-11-09T17:28:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742341_1517 (size=31105) 2024-11-09T17:28:30,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173370698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:30,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:30,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173370900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:30,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:31,001 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/977a877df997451c98f86ef05e440ccb 2024-11-09T17:28:31,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/f8446cb0abf14e3c8115faccca728cc9 is 50, key is test_row_0/B:col10/1731173308415/Put/seqid=0 2024-11-09T17:28:31,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742342_1518 (size=12151) 2024-11-09T17:28:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173371203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:31,409 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/f8446cb0abf14e3c8115faccca728cc9 2024-11-09T17:28:31,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/dfcf2d8177444890afe443a63cad13f7 is 50, key is test_row_0/C:col10/1731173308415/Put/seqid=0 2024-11-09T17:28:31,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742343_1519 (size=12151) 2024-11-09T17:28:31,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173371709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:31,817 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/dfcf2d8177444890afe443a63cad13f7 2024-11-09T17:28:31,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/977a877df997451c98f86ef05e440ccb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb 2024-11-09T17:28:31,847 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb, entries=150, sequenceid=182, filesize=30.4 K 2024-11-09T17:28:31,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/f8446cb0abf14e3c8115faccca728cc9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9 2024-11-09T17:28:31,851 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T17:28:31,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/dfcf2d8177444890afe443a63cad13f7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7 2024-11-09T17:28:31,853 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7, entries=150, sequenceid=182, filesize=11.9 K 2024-11-09T17:28:31,854 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(3040): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for e267c9faf6557bc65334d4401871cb37 in 1670ms, sequenceid=182, compaction requested=false 2024-11-09T17:28:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:31,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-11-09T17:28:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-11-09T17:28:31,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-11-09T17:28:31,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-09T17:28:31,857 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=153, table=TestAcidGuarantees in 1.9780 sec 2024-11-09T17:28:31,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-09T17:28:31,982 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 153 completed 2024-11-09T17:28:31,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees 2024-11-09T17:28:31,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:31,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:31,985 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=155, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:31,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:32,136 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:32,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=156 2024-11-09T17:28:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:32,138 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:32,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:32,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110950505096381f4b91bd033d8f48c9e071_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173310589/Put/seqid=0 2024-11-09T17:28:32,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742344_1520 (size=12304) 2024-11-09T17:28:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:32,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:32,550 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110950505096381f4b91bd033d8f48c9e071_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110950505096381f4b91bd033d8f48c9e071_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:32,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f36687f206b94b75869425a54be385e7, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:32,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f36687f206b94b75869425a54be385e7 is 175, key is test_row_0/A:col10/1731173310589/Put/seqid=0 2024-11-09T17:28:32,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742345_1521 (size=31105) 2024-11-09T17:28:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:32,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:32,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173372726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:32,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:32,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173372829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:32,954 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f36687f206b94b75869425a54be385e7 2024-11-09T17:28:32,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/46d270c3673147859b1343b76a2fc093 is 50, key is test_row_0/B:col10/1731173310589/Put/seqid=0 2024-11-09T17:28:32,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742346_1522 (size=12151) 2024-11-09T17:28:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173373030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:33,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:33,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:33,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173373334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:33,363 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/46d270c3673147859b1343b76a2fc093 2024-11-09T17:28:33,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/3cbbde013b60403a8a17af1f989b275f is 50, key is test_row_0/C:col10/1731173310589/Put/seqid=0 2024-11-09T17:28:33,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742347_1523 (size=12151) 2024-11-09T17:28:33,771 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/3cbbde013b60403a8a17af1f989b275f 2024-11-09T17:28:33,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f36687f206b94b75869425a54be385e7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7 2024-11-09T17:28:33,777 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7, entries=150, sequenceid=211, filesize=30.4 K 2024-11-09T17:28:33,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/46d270c3673147859b1343b76a2fc093 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093 2024-11-09T17:28:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,780 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T17:28:33,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/3cbbde013b60403a8a17af1f989b275f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f 2024-11-09T17:28:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,783 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f, entries=150, sequenceid=211, filesize=11.9 K 2024-11-09T17:28:33,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,784 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(3040): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e267c9faf6557bc65334d4401871cb37 in 1646ms, sequenceid=211, compaction requested=true 2024-11-09T17:28:33,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:33,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:33,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-11-09T17:28:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-11-09T17:28:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-11-09T17:28:33,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8000 sec 2024-11-09T17:28:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=155, table=TestAcidGuarantees in 1.8040 sec 2024-11-09T17:28:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:33,853 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-09T17:28:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:33,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097148f3d223d245318d2fd5725d31c5d7_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:33,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742348_1524 (size=14794) 2024-11-09T17:28:33,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173373923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173374026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-09T17:28:34,088 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 155 completed 2024-11-09T17:28:34,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$22(4386): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-09T17:28:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees 2024-11-09T17:28:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:34,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_PREPARE, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T17:28:34,090 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=157, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, locked=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T17:28:34,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T17:28:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173374229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,241 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:34,271 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097148f3d223d245318d2fd5725d31c5d7_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097148f3d223d245318d2fd5725d31c5d7_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:34,271 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2983e4b7d092426f90f1cde75186d2f4, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:34,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2983e4b7d092426f90f1cde75186d2f4 is 175, key is test_row_0/A:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:34,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742349_1525 (size=39749) 2024-11-09T17:28:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:34,394 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173374533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,546 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,675 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=222, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2983e4b7d092426f90f1cde75186d2f4 2024-11-09T17:28:34,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/6297bd6ee016475897ac99c19bf99118 is 50, key is test_row_0/B:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:34,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742350_1526 (size=12151) 2024-11-09T17:28:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:34,698 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:34,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:34,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,850 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:34,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:34,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-2 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:35,002 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:35,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] handler.RSProcedureHandler(58): pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-0 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=158 java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4114): Remote procedure failed, pid=158 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$4(MasterRpcServices.java:2571) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2566) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:35,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173375037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:35,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/6297bd6ee016475897ac99c19bf99118 2024-11-09T17:28:35,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8d5a152311134008921eb16784122ba3 is 50, key is test_row_0/C:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:35,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742351_1527 (size=12151) 2024-11-09T17:28:35,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8d5a152311134008921eb16784122ba3 2024-11-09T17:28:35,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2983e4b7d092426f90f1cde75186d2f4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4 2024-11-09T17:28:35,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4, entries=200, sequenceid=222, filesize=38.8 K 2024-11-09T17:28:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/6297bd6ee016475897ac99c19bf99118 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118 2024-11-09T17:28:35,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118, entries=150, sequenceid=222, filesize=11.9 K 2024-11-09T17:28:35,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/8d5a152311134008921eb16784122ba3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3 2024-11-09T17:28:35,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3, entries=150, sequenceid=222, filesize=11.9 K 2024-11-09T17:28:35,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for e267c9faf6557bc65334d4401871cb37 in 1252ms, sequenceid=222, compaction requested=true 2024-11-09T17:28:35,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:35,106 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:35,106 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:35,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133440 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/A is initiating minor compaction (all files) 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/B is initiating minor compaction (all files) 2024-11-09T17:28:35,107 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/A in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,107 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/B in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,107 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/39cb8c09ab234d6b8a7d7ebe5aaf9394, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.8 K 2024-11-09T17:28:35,107 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=130.3 K 2024-11-09T17:28:35,107 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4] 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 891d563360eb45d899b416887d6675e0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307282 2024-11-09T17:28:35,107 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 39cb8c09ab234d6b8a7d7ebe5aaf9394, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307282 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 977a877df997451c98f86ef05e440ccb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731173308409 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting f8446cb0abf14e3c8115faccca728cc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731173308409 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f36687f206b94b75869425a54be385e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173310589 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 46d270c3673147859b1343b76a2fc093, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173310589 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 2983e4b7d092426f90f1cde75186d2f4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:35,108 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6297bd6ee016475897ac99c19bf99118, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:35,113 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:35,116 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:35,116 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/934873fa91c4488d98c2066d637a591b is 50, key is test_row_0/B:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:35,118 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109101e40cb3d944a849959ef29b05904fd_e267c9faf6557bc65334d4401871cb37 store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:35,119 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109101e40cb3d944a849959ef29b05904fd_e267c9faf6557bc65334d4401871cb37, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:35,120 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109101e40cb3d944a849959ef29b05904fd_e267c9faf6557bc65334d4401871cb37 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:35,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742352_1528 (size=12663) 2024-11-09T17:28:35,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742353_1529 (size=4469) 2024-11-09T17:28:35,137 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#A#compaction#440 average throughput is 1.02 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:35,138 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/650e9b9fbe5745939da9138f95a14fd0 is 175, key is test_row_0/A:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:35,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742354_1530 (size=31617) 2024-11-09T17:28:35,154 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:35,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45633 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=158 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,155 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:35,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:35,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091f4938320b5f43a8b454da6aa702fab0_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173313916/Put/seqid=0 2024-11-09T17:28:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742355_1531 (size=12304) 2024-11-09T17:28:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:35,526 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/934873fa91c4488d98c2066d637a591b as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/934873fa91c4488d98c2066d637a591b 2024-11-09T17:28:35,529 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/B of e267c9faf6557bc65334d4401871cb37 into 934873fa91c4488d98c2066d637a591b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:35,529 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:35,529 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/B, priority=12, startTime=1731173315106; duration=0sec 2024-11-09T17:28:35,529 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:35,529 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:B 2024-11-09T17:28:35,529 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T17:28:35,530 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T17:28:35,530 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/C is initiating minor compaction (all files) 2024-11-09T17:28:35,530 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/C in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:35,530 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/6c4ed775b08449889bbdf58a7c2ec383, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=47.8 K 2024-11-09T17:28:35,530 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 6c4ed775b08449889bbdf58a7c2ec383, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731173307282 2024-11-09T17:28:35,530 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting dfcf2d8177444890afe443a63cad13f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731173308409 2024-11-09T17:28:35,530 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3cbbde013b60403a8a17af1f989b275f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731173310589 2024-11-09T17:28:35,531 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 8d5a152311134008921eb16784122ba3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:35,536 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:35,537 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/3cb4eccd3e40433183ea7d7205928836 is 50, key is test_row_0/C:col10/1731173313853/Put/seqid=0 2024-11-09T17:28:35,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742356_1532 (size=12663) 2024-11-09T17:28:35,543 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/3cb4eccd3e40433183ea7d7205928836 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cb4eccd3e40433183ea7d7205928836 2024-11-09T17:28:35,546 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/C of e267c9faf6557bc65334d4401871cb37 into 3cb4eccd3e40433183ea7d7205928836(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:35,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:35,546 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/C, priority=12, startTime=1731173315106; duration=0sec 2024-11-09T17:28:35,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:35,546 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:C 2024-11-09T17:28:35,557 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/650e9b9fbe5745939da9138f95a14fd0 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0 2024-11-09T17:28:35,560 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 4 (all) file(s) in e267c9faf6557bc65334d4401871cb37/A of e267c9faf6557bc65334d4401871cb37 into 650e9b9fbe5745939da9138f95a14fd0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:35,560 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:35,560 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/A, priority=12, startTime=1731173315106; duration=0sec 2024-11-09T17:28:35,560 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:35,560 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:A 2024-11-09T17:28:35,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:35,577 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091f4938320b5f43a8b454da6aa702fab0_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f4938320b5f43a8b454da6aa702fab0_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:35,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/93f4195b30cb4f2b80b35b18653ab156, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:35,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/93f4195b30cb4f2b80b35b18653ab156 is 175, key is test_row_0/A:col10/1731173313916/Put/seqid=0 2024-11-09T17:28:35,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742357_1533 (size=31105) 2024-11-09T17:28:35,982 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/93f4195b30cb4f2b80b35b18653ab156 2024-11-09T17:28:35,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/743e25cd77b7471bbf71a94b26a28056 is 50, key is test_row_0/B:col10/1731173313916/Put/seqid=0 2024-11-09T17:28:35,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742358_1534 (size=12151) 2024-11-09T17:28:36,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:36,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2496): NOT flushing TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. as already flushing 2024-11-09T17:28:36,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173376078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173376181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173376384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:36,391 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/743e25cd77b7471bbf71a94b26a28056 2024-11-09T17:28:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 is 50, key is test_row_0/C:col10/1731173313916/Put/seqid=0 2024-11-09T17:28:36,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742359_1535 (size=12151) 2024-11-09T17:28:36,582 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T17:28:36,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173376689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:36,810 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 2024-11-09T17:28:36,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/93f4195b30cb4f2b80b35b18653ab156 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156 2024-11-09T17:28:36,816 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156, entries=150, sequenceid=247, filesize=30.4 K 2024-11-09T17:28:36,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/743e25cd77b7471bbf71a94b26a28056 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056 2024-11-09T17:28:36,819 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T17:28:36,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 2024-11-09T17:28:36,823 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3, entries=150, sequenceid=247, filesize=11.9 K 2024-11-09T17:28:36,843 INFO [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for e267c9faf6557bc65334d4401871cb37 in 1688ms, sequenceid=247, compaction requested=false 2024-11-09T17:28:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:36,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c28fd47d982f:0-1 {event_type=RS_FLUSH_REGIONS, pid=158}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=158 2024-11-09T17:28:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster(4106): Remote procedure done, pid=158 2024-11-09T17:28:36,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=158, resume processing ppid=157 2024-11-09T17:28:36,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, ppid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7540 sec 2024-11-09T17:28:36,846 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=157, state=SUCCESS; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=157, table=TestAcidGuarantees in 2.7570 sec 2024-11-09T17:28:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:37,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-09T17:28:37,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:37,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:37,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:37,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:37,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:37,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:37,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095743208b382f4c3da019cc2ef712591b_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:37,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742360_1536 (size=12404) 2024-11-09T17:28:37,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173377250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:37,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:37,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173377353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:37,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:37,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173377557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:37,560 DEBUG [Thread-2131 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x503a7d2e to 127.0.0.1:57222 2024-11-09T17:28:37,560 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:37,561 DEBUG [Thread-2127 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5bb75907 to 127.0.0.1:57222 2024-11-09T17:28:37,561 DEBUG [Thread-2127 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:37,562 DEBUG [Thread-2123 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5f1754bc to 127.0.0.1:57222 2024-11-09T17:28:37,562 DEBUG [Thread-2123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:37,563 DEBUG [Thread-2129 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2c1d3a95 to 127.0.0.1:57222 2024-11-09T17:28:37,563 DEBUG [Thread-2129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:37,564 DEBUG [Thread-2125 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d9113f3 to 127.0.0.1:57222 2024-11-09T17:28:37,564 DEBUG [Thread-2125 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:37,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:37,610 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095743208b382f4c3da019cc2ef712591b_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095743208b382f4c3da019cc2ef712591b_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:37,611 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f1d2ef0e22084df7870ca50f54d5fde4, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:37,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f1d2ef0e22084df7870ca50f54d5fde4 is 175, key is test_row_0/A:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:37,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742361_1537 (size=31205) 2024-11-09T17:28:37,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173377859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f1d2ef0e22084df7870ca50f54d5fde4 2024-11-09T17:28:38,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c5a6262895a14658abc4c969e57d8ae5 is 50, key is test_row_0/B:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:38,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742362_1538 (size=12251) 2024-11-09T17:28:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60404 deadline: 1731173378058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,059 DEBUG [Thread-2118 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18205 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:38,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60398 deadline: 1731173378090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,091 DEBUG [Thread-2114 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18269 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:38,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60376 deadline: 1731173378108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,109 DEBUG [Thread-2116 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18255 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_0' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:38,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60390 deadline: 1731173378120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,122 DEBUG [Thread-2112 {}] client.RpcRetryingCallerImpl(129): Call exception, tries=8, retries=16, started=18268 ms ago, cancelled=false, msg=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) , details=row 'test_row_1' on table 'TestAcidGuarantees' at region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., hostname=c28fd47d982f,45633,1731173138129, seqNum=5, see https://s.apache.org/timeout, exception=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) at java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) at java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:280) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.handleRemoteException(ProtobufUtil.java:265) at org.apache.hadoop.hbase.client.RegionServerCallable.call(RegionServerCallable.java:133) at org.apache.hadoop.hbase.client.RpcRetryingCallerImpl.callWithRetries(RpcRetryingCallerImpl.java:104) at org.apache.hadoop.hbase.client.HTable.lambda$put$3(HTable.java:578) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.client.HTable.put(HTable.java:565) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.doAnAction(AcidGuaranteesTestTool.java:169) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:149) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.RegionTooBusyException): org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T17:28:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-09T17:28:38,195 INFO [Thread-2122 {}] client.HBaseAdmin$TableFuture(3751): Operation: FLUSH, Table Name: default:TestAcidGuarantees, procId: 157 completed 2024-11-09T17:28:38,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(5069): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5067) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$9(HRegion.java:3229) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3222) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3072) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:3035) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-09T17:28:38,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60414 deadline: 1731173378363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:38,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c5a6262895a14658abc4c969e57d8ae5 2024-11-09T17:28:38,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d15907a9bd8b4972a632301a50243149 is 50, key is test_row_0/C:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:38,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742363_1539 (size=12251) 2024-11-09T17:28:38,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d15907a9bd8b4972a632301a50243149 2024-11-09T17:28:38,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/f1d2ef0e22084df7870ca50f54d5fde4 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4 2024-11-09T17:28:38,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4, entries=150, sequenceid=262, filesize=30.5 K 2024-11-09T17:28:38,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/c5a6262895a14658abc4c969e57d8ae5 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5 2024-11-09T17:28:38,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T17:28:38,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/d15907a9bd8b4972a632301a50243149 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149 2024-11-09T17:28:38,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T17:28:38,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for e267c9faf6557bc65334d4401871cb37 in 1679ms, sequenceid=262, compaction requested=true 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:38,876 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:38,876 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e267c9faf6557bc65334d4401871cb37:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T17:28:38,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:38,877 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93927 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:38,877 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/A is initiating minor compaction (all files) 2024-11-09T17:28:38,877 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:38,877 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/A in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:38,878 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/B is initiating minor compaction (all files) 2024-11-09T17:28:38,878 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/B in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:38,878 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=91.7 K 2024-11-09T17:28:38,878 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:38,878 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/934873fa91c4488d98c2066d637a591b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=36.2 K 2024-11-09T17:28:38,878 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. files: [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4] 2024-11-09T17:28:38,878 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 934873fa91c4488d98c2066d637a591b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:38,878 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 650e9b9fbe5745939da9138f95a14fd0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:38,878 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting 93f4195b30cb4f2b80b35b18653ab156, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731173313916 2024-11-09T17:28:38,879 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 743e25cd77b7471bbf71a94b26a28056, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731173313916 2024-11-09T17:28:38,879 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] compactions.Compactor(224): Compacting f1d2ef0e22084df7870ca50f54d5fde4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731173316047 2024-11-09T17:28:38,879 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting c5a6262895a14658abc4c969e57d8ae5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731173316047 2024-11-09T17:28:38,886 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(322): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:38,887 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:38,887 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(626): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090b95e3fc3ea64f59bb740f80da693659_e267c9faf6557bc65334d4401871cb37 store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:38,887 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/fb6ef88a28ed4bd2bc6772b0d5a0051f is 50, key is test_row_0/B:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:38,890 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(647): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090b95e3fc3ea64f59bb740f80da693659_e267c9faf6557bc65334d4401871cb37, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:38,890 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] mob.DefaultMobStoreCompactor(658): Aborting writer for hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090b95e3fc3ea64f59bb740f80da693659_e267c9faf6557bc65334d4401871cb37 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:38,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742364_1540 (size=12865) 2024-11-09T17:28:38,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742365_1541 (size=4469) 2024-11-09T17:28:39,297 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#A#compaction#450 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:39,298 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d6fd5dc8e9b34973a8a3bb8192456f02 is 175, key is test_row_0/A:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:39,303 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/fb6ef88a28ed4bd2bc6772b0d5a0051f as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/fb6ef88a28ed4bd2bc6772b0d5a0051f 2024-11-09T17:28:39,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742366_1542 (size=31819) 2024-11-09T17:28:39,309 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/B of e267c9faf6557bc65334d4401871cb37 into fb6ef88a28ed4bd2bc6772b0d5a0051f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:39,309 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:39,309 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/B, priority=13, startTime=1731173318876; duration=0sec 2024-11-09T17:28:39,309 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T17:28:39,309 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:B 2024-11-09T17:28:39,309 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T17:28:39,310 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T17:28:39,310 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1540): e267c9faf6557bc65334d4401871cb37/C is initiating minor compaction (all files) 2024-11-09T17:28:39,310 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2351): Starting compaction of e267c9faf6557bc65334d4401871cb37/C in TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:39,310 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cb4eccd3e40433183ea7d7205928836, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149] into tmpdir=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp, totalSize=36.2 K 2024-11-09T17:28:39,311 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 3cb4eccd3e40433183ea7d7205928836, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1731173312720 2024-11-09T17:28:39,311 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting 2aa2bcfdc7d349eaa963ae1b047d1eb3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731173313916 2024-11-09T17:28:39,312 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] compactions.Compactor(224): Compacting d15907a9bd8b4972a632301a50243149, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731173316047 2024-11-09T17:28:39,318 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e267c9faf6557bc65334d4401871cb37#C#compaction#451 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T17:28:39,319 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/20f98135445a4371b0a88517c1b4d8eb is 50, key is test_row_0/C:col10/1731173316077/Put/seqid=0 2024-11-09T17:28:39,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742367_1543 (size=12865) 2024-11-09T17:28:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45633 {}] regionserver.HRegion(8581): Flush requested on e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:39,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-09T17:28:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:39,369 DEBUG [Thread-2120 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d7fe93b to 127.0.0.1:57222 2024-11-09T17:28:39,369 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:39,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:39,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:39,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961e28a00dca0440bb9d3fbdf3f18ca7b_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173319367/Put/seqid=0 2024-11-09T17:28:39,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742368_1544 (size=12454) 2024-11-09T17:28:39,717 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/d6fd5dc8e9b34973a8a3bb8192456f02 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d6fd5dc8e9b34973a8a3bb8192456f02 2024-11-09T17:28:39,721 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/A of e267c9faf6557bc65334d4401871cb37 into d6fd5dc8e9b34973a8a3bb8192456f02(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:39,721 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:39,721 INFO [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/A, priority=13, startTime=1731173318876; duration=0sec 2024-11-09T17:28:39,721 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:39,721 DEBUG [RS:0;c28fd47d982f:45633-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:A 2024-11-09T17:28:39,726 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/20f98135445a4371b0a88517c1b4d8eb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/20f98135445a4371b0a88517c1b4d8eb 2024-11-09T17:28:39,729 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HStore(1336): Completed compaction of 3 (all) file(s) in e267c9faf6557bc65334d4401871cb37/C of e267c9faf6557bc65334d4401871cb37 into 20f98135445a4371b0a88517c1b4d8eb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T17:28:39,729 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.HRegion(2381): Compaction status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:39,729 INFO [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37., storeName=e267c9faf6557bc65334d4401871cb37/C, priority=13, startTime=1731173318876; duration=0sec 2024-11-09T17:28:39,729 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T17:28:39,729 DEBUG [RS:0;c28fd47d982f:45633-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e267c9faf6557bc65334d4401871cb37:C 2024-11-09T17:28:39,787 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:39,794 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110961e28a00dca0440bb9d3fbdf3f18ca7b_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961e28a00dca0440bb9d3fbdf3f18ca7b_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:39,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2fe78b87bf1947b4a5c6fc9cb49c780e, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:39,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2fe78b87bf1947b4a5c6fc9cb49c780e is 175, key is test_row_0/A:col10/1731173319367/Put/seqid=0 2024-11-09T17:28:39,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742369_1545 (size=31255) 2024-11-09T17:28:40,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2fe78b87bf1947b4a5c6fc9cb49c780e 2024-11-09T17:28:40,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/03d1a1f835374324a3b61de03a831b87 is 50, key is test_row_0/B:col10/1731173319367/Put/seqid=0 2024-11-09T17:28:40,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742370_1546 (size=12301) 2024-11-09T17:28:40,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/03d1a1f835374324a3b61de03a831b87 2024-11-09T17:28:40,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1f3c8858a3024ffc9ee0721511e31efa is 50, key is test_row_0/C:col10/1731173319367/Put/seqid=0 2024-11-09T17:28:40,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742371_1547 (size=12301) 2024-11-09T17:28:41,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1f3c8858a3024ffc9ee0721511e31efa 2024-11-09T17:28:41,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/2fe78b87bf1947b4a5c6fc9cb49c780e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2fe78b87bf1947b4a5c6fc9cb49c780e 2024-11-09T17:28:41,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2fe78b87bf1947b4a5c6fc9cb49c780e, entries=150, sequenceid=288, filesize=30.5 K 2024-11-09T17:28:41,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/03d1a1f835374324a3b61de03a831b87 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/03d1a1f835374324a3b61de03a831b87 2024-11-09T17:28:41,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/03d1a1f835374324a3b61de03a831b87, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T17:28:41,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/1f3c8858a3024ffc9ee0721511e31efa as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1f3c8858a3024ffc9ee0721511e31efa 2024-11-09T17:28:41,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1f3c8858a3024ffc9ee0721511e31efa, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T17:28:41,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for e267c9faf6557bc65334d4401871cb37 in 1688ms, sequenceid=288, compaction requested=false 2024-11-09T17:28:41,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:43,607 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:46491,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:46,499 ERROR [LeaseRenewer:jenkins@localhost:46491 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:46491,5,PEWorkerGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T17:28:48,108 DEBUG [Thread-2118 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58971172 to 127.0.0.1:57222 2024-11-09T17:28:48,108 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:48,110 DEBUG [Thread-2114 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e047c09 to 127.0.0.1:57222 2024-11-09T17:28:48,110 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:48,143 DEBUG [Thread-2116 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60d631a3 to 127.0.0.1:57222 2024-11-09T17:28:48,143 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:48,208 DEBUG [Thread-2112 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x103dfc6e to 127.0.0.1:57222 2024-11-09T17:28:48,208 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 172 2024-11-09T17:28:48,209 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6248 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6060 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6220 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6243 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 6069 2024-11-09T17:28:48,210 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T17:28:48,210 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:28:48,210 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06094c70 to 127.0.0.1:57222 2024-11-09T17:28:48,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:48,211 INFO [Time-limited test {}] client.HBaseAdmin$18(967): Started disable of TestAcidGuarantees 2024-11-09T17:28:48,211 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$13(2755): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-09T17:28:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=159, state=RUNNABLE:DISABLE_TABLE_PREPARE; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:48,215 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173328214"}]},"ts":"1731173328214"} 2024-11-09T17:28:48,216 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T17:28:48,233 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T17:28:48,235 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T17:28:48,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, UNASSIGN}] 2024-11-09T17:28:48,238 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=161, ppid=160, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, UNASSIGN 2024-11-09T17:28:48,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=161 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=CLOSING, regionLocation=c28fd47d982f,45633,1731173138129 2024-11-09T17:28:48,241 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(338): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T17:28:48,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; CloseRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129}] 2024-11-09T17:28:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:48,393 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to c28fd47d982f,45633,1731173138129 2024-11-09T17:28:48,394 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(124): Close e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:48,394 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(138): Unassign region: split region: false: evictCache: false 2024-11-09T17:28:48,395 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1681): Closing e267c9faf6557bc65334d4401871cb37, disabling compactions & flushes 2024-11-09T17:28:48,395 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1703): Closing region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:48,395 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1724): Waiting without time limit for close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:48,395 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1791): Acquired close lock on TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. after waiting 0 ms 2024-11-09T17:28:48,395 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1801): Updates disabled for region TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:48,395 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(2837): Flushing e267c9faf6557bc65334d4401871cb37 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-09T17:28:48,395 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=A 2024-11-09T17:28:48,396 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:48,396 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=B 2024-11-09T17:28:48,396 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:48,396 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e267c9faf6557bc65334d4401871cb37, store=C 2024-11-09T17:28:48,396 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T17:28:48,407 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099404d468f80e4a72b60196105b463f4d_e267c9faf6557bc65334d4401871cb37 is 50, key is test_row_0/A:col10/1731173328141/Put/seqid=0 2024-11-09T17:28:48,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742372_1548 (size=9914) 2024-11-09T17:28:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:48,811 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T17:28:48,816 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099404d468f80e4a72b60196105b463f4d_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099404d468f80e4a72b60196105b463f4d_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:48,817 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/ed0edd46455247a78a5d643f841f113e, store: [table=TestAcidGuarantees family=A region=e267c9faf6557bc65334d4401871cb37] 2024-11-09T17:28:48,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:48,818 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/ed0edd46455247a78a5d643f841f113e is 175, key is test_row_0/A:col10/1731173328141/Put/seqid=0 2024-11-09T17:28:48,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742373_1549 (size=22561) 2024-11-09T17:28:49,225 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/ed0edd46455247a78a5d643f841f113e 2024-11-09T17:28:49,238 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/9ebb53b5ae684472a1f5fd00ced43071 is 50, key is test_row_0/B:col10/1731173328141/Put/seqid=0 2024-11-09T17:28:49,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742374_1550 (size=9857) 2024-11-09T17:28:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:49,644 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/9ebb53b5ae684472a1f5fd00ced43071 2024-11-09T17:28:49,659 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/88537878d4a84ab19e7a55f97e5d3b79 is 50, key is test_row_0/C:col10/1731173328141/Put/seqid=0 2024-11-09T17:28:49,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742375_1551 (size=9857) 2024-11-09T17:28:50,062 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/88537878d4a84ab19e7a55f97e5d3b79 2024-11-09T17:28:50,066 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/A/ed0edd46455247a78a5d643f841f113e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/ed0edd46455247a78a5d643f841f113e 2024-11-09T17:28:50,068 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/ed0edd46455247a78a5d643f841f113e, entries=100, sequenceid=297, filesize=22.0 K 2024-11-09T17:28:50,069 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/B/9ebb53b5ae684472a1f5fd00ced43071 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/9ebb53b5ae684472a1f5fd00ced43071 2024-11-09T17:28:50,072 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/9ebb53b5ae684472a1f5fd00ced43071, entries=100, sequenceid=297, filesize=9.6 K 2024-11-09T17:28:50,072 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/.tmp/C/88537878d4a84ab19e7a55f97e5d3b79 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/88537878d4a84ab19e7a55f97e5d3b79 2024-11-09T17:28:50,075 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/88537878d4a84ab19e7a55f97e5d3b79, entries=100, sequenceid=297, filesize=9.6 K 2024-11-09T17:28:50,076 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(3040): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for e267c9faf6557bc65334d4401871cb37 in 1680ms, sequenceid=297, compaction requested=true 2024-11-09T17:28:50,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4] to archive 2024-11-09T17:28:50,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:50,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1e5ea8bfd3ab49ae891532487eb070d2 2024-11-09T17:28:50,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/11afc75b8caf4acdb644277ec55f66b5 2024-11-09T17:28:50,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/34bcf2e2b5554ff98ab679dc0ec316e8 2024-11-09T17:28:50,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1fbba25eec5f4f48af37c92310538c5e 2024-11-09T17:28:50,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d21f89e6a21b46f291ae4aba0c1a99b8 2024-11-09T17:28:50,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/207552f0f52f4367818b2746e7d96289 2024-11-09T17:28:50,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/1132ea16c7c74cba92521e9d4f9a82bf 2024-11-09T17:28:50,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/0e4c8c8c5be74265b942cae49001b9be 2024-11-09T17:28:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/cb324b8bf36f45f581dd4fef587fd67d 2024-11-09T17:28:50,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/dd6ad85a1f0c471f98686bf8938d98e6 2024-11-09T17:28:50,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/3398cc56dec0427f92819d8590769dee 2024-11-09T17:28:50,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/891d563360eb45d899b416887d6675e0 2024-11-09T17:28:50,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/977a877df997451c98f86ef05e440ccb 2024-11-09T17:28:50,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f36687f206b94b75869425a54be385e7 2024-11-09T17:28:50,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2983e4b7d092426f90f1cde75186d2f4 2024-11-09T17:28:50,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/650e9b9fbe5745939da9138f95a14fd0 2024-11-09T17:28:50,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/93f4195b30cb4f2b80b35b18653ab156 2024-11-09T17:28:50,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/f1d2ef0e22084df7870ca50f54d5fde4 2024-11-09T17:28:50,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/e204d1cb6f174056b79f637a65f14d8a, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/af5d30135d47414fbbe9983188f519f5, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/39cb8c09ab234d6b8a7d7ebe5aaf9394, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/934873fa91c4488d98c2066d637a591b, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5] to archive 2024-11-09T17:28:50,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:50,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/200aeba9aa0b448d87a53899347b6864 2024-11-09T17:28:50,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7d5e02fcd9d447678abb96963a740bd6 2024-11-09T17:28:50,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/e204d1cb6f174056b79f637a65f14d8a to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/e204d1cb6f174056b79f637a65f14d8a 2024-11-09T17:28:50,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c1eaab287d404722a54e8e6a019f5be0 2024-11-09T17:28:50,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/35da710add2c41a1a5ab7f96ff211611 2024-11-09T17:28:50,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ebc01bb6a1a248ea8208e9e0651c2899 2024-11-09T17:28:50,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/af5d30135d47414fbbe9983188f519f5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/af5d30135d47414fbbe9983188f519f5 2024-11-09T17:28:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/7ca8b7cd2046440889bb991281395617 2024-11-09T17:28:50,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/5c04fbfa62e94163821f0349afba5afd 2024-11-09T17:28:50,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/ee2b1b33ed0f4e9789935fdc62836a43 2024-11-09T17:28:50,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/39cb8c09ab234d6b8a7d7ebe5aaf9394 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/39cb8c09ab234d6b8a7d7ebe5aaf9394 2024-11-09T17:28:50,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c74f62525974432d807add5cace57276 2024-11-09T17:28:50,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/f8446cb0abf14e3c8115faccca728cc9 2024-11-09T17:28:50,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/46d270c3673147859b1343b76a2fc093 2024-11-09T17:28:50,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/934873fa91c4488d98c2066d637a591b to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/934873fa91c4488d98c2066d637a591b 2024-11-09T17:28:50,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/6297bd6ee016475897ac99c19bf99118 2024-11-09T17:28:50,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/743e25cd77b7471bbf71a94b26a28056 2024-11-09T17:28:50,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/c5a6262895a14658abc4c969e57d8ae5 2024-11-09T17:28:50,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] regionserver.HStore(2316): Moving the files [hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1ac9306b6e6a4948aa63a03c9c70ec22, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/0129117af7c34ab9adcc71c29cde90c7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/6c4ed775b08449889bbdf58a7c2ec383, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cb4eccd3e40433183ea7d7205928836, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149] to archive 2024-11-09T17:28:50,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T17:28:50,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/234bd444aafc4dc18a804fc580772887 2024-11-09T17:28:50,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2b9d65901e214ff7a8a7c5be453f8585 2024-11-09T17:28:50,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1ac9306b6e6a4948aa63a03c9c70ec22 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1ac9306b6e6a4948aa63a03c9c70ec22 2024-11-09T17:28:50,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/10b6dcee6f064ec6b2a56c811cbd5d62 2024-11-09T17:28:50,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1144957abf4c445f850b73bb351e3a12 2024-11-09T17:28:50,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/7d32d7951bd94c1fa7357f25d10ce2a3 2024-11-09T17:28:50,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/0129117af7c34ab9adcc71c29cde90c7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/0129117af7c34ab9adcc71c29cde90c7 2024-11-09T17:28:50,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/ace2a76ba58c45a7afbf06b8d07ce778 2024-11-09T17:28:50,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/5ca4314d472d414ca1b84a9c4060c8e0 2024-11-09T17:28:50,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8aa266f38b5d4300ad4e963da3cb512f 2024-11-09T17:28:50,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/6c4ed775b08449889bbdf58a7c2ec383 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/6c4ed775b08449889bbdf58a7c2ec383 2024-11-09T17:28:50,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d2696b24820543cd82fe60f7b06f17c8 2024-11-09T17:28:50,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/dfcf2d8177444890afe443a63cad13f7 2024-11-09T17:28:50,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cbbde013b60403a8a17af1f989b275f 2024-11-09T17:28:50,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cb4eccd3e40433183ea7d7205928836 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/3cb4eccd3e40433183ea7d7205928836 2024-11-09T17:28:50,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/8d5a152311134008921eb16784122ba3 2024-11-09T17:28:50,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/2aa2bcfdc7d349eaa963ae1b047d1eb3 2024-11-09T17:28:50,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/d15907a9bd8b4972a632301a50243149 2024-11-09T17:28:50,120 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits/300.seqid, newMaxSeqId=300, maxSeqId=4 2024-11-09T17:28:50,121 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1922): Closed TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37. 2024-11-09T17:28:50,121 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] regionserver.HRegion(1635): Region close journal for e267c9faf6557bc65334d4401871cb37: 2024-11-09T17:28:50,122 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION, pid=162}] handler.UnassignRegionHandler(170): Closed e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,122 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=161 updating hbase:meta row=e267c9faf6557bc65334d4401871cb37, regionState=CLOSED 2024-11-09T17:28:50,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-11-09T17:28:50,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; CloseRegionProcedure e267c9faf6557bc65334d4401871cb37, server=c28fd47d982f,45633,1731173138129 in 1.8820 sec 2024-11-09T17:28:50,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=161, resume processing ppid=160 2024-11-09T17:28:50,124 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, ppid=160, state=SUCCESS; TransitRegionStateProcedure table=TestAcidGuarantees, region=e267c9faf6557bc65334d4401871cb37, UNASSIGN in 1.8860 sec 2024-11-09T17:28:50,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=160, resume processing ppid=159 2024-11-09T17:28:50,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=160, ppid=159, state=SUCCESS; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8890 sec 2024-11-09T17:28:50,126 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731173330125"}]},"ts":"1731173330125"} 2024-11-09T17:28:50,126 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(1655): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T17:28:50,170 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T17:28:50,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, state=SUCCESS; DisableTableProcedure table=TestAcidGuarantees in 1.9590 sec 2024-11-09T17:28:50,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=159 2024-11-09T17:28:50,322 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DISABLE, Table Name: default:TestAcidGuarantees, procId: 159 completed 2024-11-09T17:28:50,323 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.HMaster$5(2505): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-09T17:28:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,325 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=163, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-09T17:28:50,326 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=163, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,330 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,334 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C, FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits] 2024-11-09T17:28:50,340 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2fe78b87bf1947b4a5c6fc9cb49c780e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/2fe78b87bf1947b4a5c6fc9cb49c780e 2024-11-09T17:28:50,342 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d6fd5dc8e9b34973a8a3bb8192456f02 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/d6fd5dc8e9b34973a8a3bb8192456f02 2024-11-09T17:28:50,344 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/ed0edd46455247a78a5d643f841f113e to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/A/ed0edd46455247a78a5d643f841f113e 2024-11-09T17:28:50,348 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/03d1a1f835374324a3b61de03a831b87 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/03d1a1f835374324a3b61de03a831b87 2024-11-09T17:28:50,350 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/9ebb53b5ae684472a1f5fd00ced43071 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/9ebb53b5ae684472a1f5fd00ced43071 2024-11-09T17:28:50,352 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/fb6ef88a28ed4bd2bc6772b0d5a0051f to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/B/fb6ef88a28ed4bd2bc6772b0d5a0051f 2024-11-09T17:28:50,356 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1f3c8858a3024ffc9ee0721511e31efa to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/1f3c8858a3024ffc9ee0721511e31efa 2024-11-09T17:28:50,358 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/20f98135445a4371b0a88517c1b4d8eb to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/20f98135445a4371b0a88517c1b4d8eb 2024-11-09T17:28:50,360 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/88537878d4a84ab19e7a55f97e5d3b79 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/C/88537878d4a84ab19e7a55f97e5d3b79 2024-11-09T17:28:50,363 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits/300.seqid to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37/recovered.edits/300.seqid 2024-11-09T17:28:50,363 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/default/TestAcidGuarantees/e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,364 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T17:28:50,364 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:28:50,365 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T17:28:50,369 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090702cbfe03684eef9dc8b1b8950b18da_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090702cbfe03684eef9dc8b1b8950b18da_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,370 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109086364ae16524401ab79dff098b2a225_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109086364ae16524401ab79dff098b2a225_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,372 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f4938320b5f43a8b454da6aa702fab0_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091f4938320b5f43a8b454da6aa702fab0_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,374 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094f60a45f149544c194321080d2005f99_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094f60a45f149544c194321080d2005f99_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,376 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110950505096381f4b91bd033d8f48c9e071_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110950505096381f4b91bd033d8f48c9e071_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,378 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095743208b382f4c3da019cc2ef712591b_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095743208b382f4c3da019cc2ef712591b_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,380 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095dd1686d372d4ff984ae84f0d7834991_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095dd1686d372d4ff984ae84f0d7834991_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,382 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961e28a00dca0440bb9d3fbdf3f18ca7b_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110961e28a00dca0440bb9d3fbdf3f18ca7b_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,385 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097148f3d223d245318d2fd5725d31c5d7_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097148f3d223d245318d2fd5725d31c5d7_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,387 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989a277aee605449d9366d423519ce299_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110989a277aee605449d9366d423519ce299_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,388 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099404d468f80e4a72b60196105b463f4d_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099404d468f80e4a72b60196105b463f4d_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,390 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4e78f3477644974b08ea792c0449fd5_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a4e78f3477644974b08ea792c0449fd5_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,391 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cc642dc5a36a45d7966aabfb0977e518_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109cc642dc5a36a45d7966aabfb0977e518_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,393 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e025fc66c58d49b598ed17f59f07febb_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e025fc66c58d49b598ed17f59f07febb_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed08f25647af451c8606d93da97b9424_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ed08f25647af451c8606d93da97b9424_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,396 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f4b484af4b0d4997b236c340234b3e23_e267c9faf6557bc65334d4401871cb37 to hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109f4b484af4b0d4997b236c340234b3e23_e267c9faf6557bc65334d4401871cb37 2024-11-09T17:28:50,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T17:28:50,399 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=163, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,401 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T17:28:50,403 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T17:28:50,404 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=163, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,404 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T17:28:50,404 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731173330404"}]},"ts":"9223372036854775807"} 2024-11-09T17:28:50,406 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1808): Deleted 1 regions from META 2024-11-09T17:28:50,406 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(1809): Deleted regions: [{ENCODED => e267c9faf6557bc65334d4401871cb37, NAME => 'TestAcidGuarantees,,1731173294296.e267c9faf6557bc65334d4401871cb37.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T17:28:50,406 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T17:28:50,406 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(2113): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731173330406"}]},"ts":"9223372036854775807"} 2024-11-09T17:28:50,408 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(1678): Deleted table TestAcidGuarantees state from META 2024-11-09T17:28:50,417 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=163, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, locked=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T17:28:50,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=163, state=SUCCESS; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-11-09T17:28:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44083 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-09T17:28:50,427 INFO [Time-limited test {}] client.HBaseAdmin$TableFuture(3751): Operation: DELETE, Table Name: default:TestAcidGuarantees, procId: 163 completed 2024-11-09T17:28:50,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithBasicPolicy#testMobGetAtomicity Thread=235 (was 239), OpenFileDescriptor=447 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=250 (was 295), ProcessCount=11 (was 11), AvailableMemoryMB=5812 (was 5825) 2024-11-09T17:28:50,441 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-09T17:28:50,441 INFO [Time-limited test {}] client.ConnectionImplementation(2127): Closing master protocol: MasterService 2024-11-09T17:28:50,441 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e541e88 to 127.0.0.1:57222 2024-11-09T17:28:50,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:50,441 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T17:28:50,441 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1111253708, stopped=false 2024-11-09T17:28:50,441 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=c28fd47d982f,44083,1731173137386 2024-11-09T17:28:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T17:28:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T17:28:50,450 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-09T17:28:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:28:50,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:28:50,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:50,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T17:28:50,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T17:28:50,451 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'c28fd47d982f,45633,1731173138129' ***** 2024-11-09T17:28:50,451 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-09T17:28:50,451 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T17:28:50,451 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-09T17:28:50,451 INFO [RS:0;c28fd47d982f:45633 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(3579): Received CLOSE for 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1224): stopping server c28fd47d982f,45633,1731173138129 2024-11-09T17:28:50,452 DEBUG [RS:0;c28fd47d982f:45633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T17:28:50,452 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T17:28:50,453 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 8909151a42b30d353536c39084e3c0b3, disabling compactions & flushes 2024-11-09T17:28:50,453 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. after waiting 0 ms 2024-11-09T17:28:50,453 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:28:50,453 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1603): Online Regions={8909151a42b30d353536c39084e3c0b3=hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3., 1588230740=hbase:meta,,1.1588230740} 2024-11-09T17:28:50,453 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 8909151a42b30d353536c39084e3c0b3 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-09T17:28:50,453 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T17:28:50,453 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T17:28:50,453 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=20.55 KB heapSize=35.87 KB 2024-11-09T17:28:50,456 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:28:50,470 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/.tmp/info/2e971c7c5840471a9289895c9a2330d7 is 45, key is default/info:d/1731173143891/Put/seqid=0 2024-11-09T17:28:50,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742376_1552 (size=5037) 2024-11-09T17:28:50,479 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/info/a55aff30e12e40588956428b8c8c9698 is 143, key is hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3./info:regioninfo/1731173143757/Put/seqid=0 2024-11-09T17:28:50,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742377_1553 (size=7725) 2024-11-09T17:28:50,538 INFO [regionserver/c28fd47d982f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T17:28:50,657 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:28:50,857 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 8909151a42b30d353536c39084e3c0b3 2024-11-09T17:28:50,875 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/.tmp/info/2e971c7c5840471a9289895c9a2330d7 2024-11-09T17:28:50,883 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/info/a55aff30e12e40588956428b8c8c9698 2024-11-09T17:28:50,884 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/.tmp/info/2e971c7c5840471a9289895c9a2330d7 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/info/2e971c7c5840471a9289895c9a2330d7 2024-11-09T17:28:50,890 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/info/2e971c7c5840471a9289895c9a2330d7, entries=2, sequenceid=6, filesize=4.9 K 2024-11-09T17:28:50,891 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 8909151a42b30d353536c39084e3c0b3 in 438ms, sequenceid=6, compaction requested=false 2024-11-09T17:28:50,895 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/namespace/8909151a42b30d353536c39084e3c0b3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-09T17:28:50,896 INFO [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:28:50,896 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 8909151a42b30d353536c39084e3c0b3: 2024-11-09T17:28:50,896 DEBUG [RS_CLOSE_REGION-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731173142433.8909151a42b30d353536c39084e3c0b3. 2024-11-09T17:28:50,906 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/rep_barrier/04cf195c814c44bab7efeffac4e98ae9 is 102, key is TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2./rep_barrier:/1731173173761/DeleteFamily/seqid=0 2024-11-09T17:28:50,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742378_1554 (size=6025) 2024-11-09T17:28:50,947 INFO [regionserver/c28fd47d982f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T17:28:50,947 INFO [regionserver/c28fd47d982f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T17:28:51,058 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-09T17:28:51,258 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-09T17:28:51,310 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/rep_barrier/04cf195c814c44bab7efeffac4e98ae9 2024-11-09T17:28:51,336 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/table/bc9801984efd481c80e2360e405c4073 is 96, key is TestAcidGuarantees,,1731173144161.d0b59f7cee8e92155959d9919114a0a2./table:/1731173173761/DeleteFamily/seqid=0 2024-11-09T17:28:51,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742379_1555 (size=5942) 2024-11-09T17:28:51,458 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1599): Waiting on 1 regions to close 2024-11-09T17:28:51,458 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T17:28:51,459 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-09T17:28:51,659 DEBUG [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-09T17:28:51,741 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.08 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/table/bc9801984efd481c80e2360e405c4073 2024-11-09T17:28:51,750 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/info/a55aff30e12e40588956428b8c8c9698 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/info/a55aff30e12e40588956428b8c8c9698 2024-11-09T17:28:51,756 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/info/a55aff30e12e40588956428b8c8c9698, entries=22, sequenceid=93, filesize=7.5 K 2024-11-09T17:28:51,757 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/rep_barrier/04cf195c814c44bab7efeffac4e98ae9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/rep_barrier/04cf195c814c44bab7efeffac4e98ae9 2024-11-09T17:28:51,761 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/rep_barrier/04cf195c814c44bab7efeffac4e98ae9, entries=6, sequenceid=93, filesize=5.9 K 2024-11-09T17:28:51,761 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/.tmp/table/bc9801984efd481c80e2360e405c4073 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/table/bc9801984efd481c80e2360e405c4073 2024-11-09T17:28:51,765 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/table/bc9801984efd481c80e2360e405c4073, entries=9, sequenceid=93, filesize=5.8 K 2024-11-09T17:28:51,765 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~20.55 KB/21040, heapSize ~35.82 KB/36680, currentSize=0 B/0 for 1588230740 in 1312ms, sequenceid=93, compaction requested=false 2024-11-09T17:28:51,770 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/data/hbase/meta/1588230740/recovered.edits/96.seqid, newMaxSeqId=96, maxSeqId=1 2024-11-09T17:28:51,770 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T17:28:51,770 INFO [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-09T17:28:51,770 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-09T17:28:51,771 DEBUG [RS_CLOSE_META-regionserver/c28fd47d982f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T17:28:51,859 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1250): stopping server c28fd47d982f,45633,1731173138129; all regions closed. 2024-11-09T17:28:51,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741834_1010 (size=26050) 2024-11-09T17:28:51,873 DEBUG [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/oldWALs 2024-11-09T17:28:51,873 INFO [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL c28fd47d982f%2C45633%2C1731173138129.meta:.meta(num 1731173142127) 2024-11-09T17:28:51,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741832_1008 (size=12329849) 2024-11-09T17:28:51,879 DEBUG [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/oldWALs 2024-11-09T17:28:51,879 INFO [RS:0;c28fd47d982f:45633 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL c28fd47d982f%2C45633%2C1731173138129:(num 1731173141101) 2024-11-09T17:28:51,879 DEBUG [RS:0;c28fd47d982f:45633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:51,879 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T17:28:51,880 INFO [RS:0;c28fd47d982f:45633 {}] hbase.ChoreService(370): Chore service for: regionserver/c28fd47d982f:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T17:28:51,880 INFO [regionserver/c28fd47d982f:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T17:28:51,881 INFO [RS:0;c28fd47d982f:45633 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.3:45633 2024-11-09T17:28:51,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c28fd47d982f,45633,1731173138129 2024-11-09T17:28:51,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T17:28:51,937 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c28fd47d982f,45633,1731173138129] 2024-11-09T17:28:51,937 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing c28fd47d982f,45633,1731173138129; numProcessing=1 2024-11-09T17:28:51,945 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/c28fd47d982f,45633,1731173138129 already deleted, retry=false 2024-11-09T17:28:51,945 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; c28fd47d982f,45633,1731173138129 expired; onlineServers=0 2024-11-09T17:28:51,945 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'c28fd47d982f,44083,1731173137386' ***** 2024-11-09T17:28:51,945 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T17:28:51,946 DEBUG [M:0;c28fd47d982f:44083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ad2c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c28fd47d982f/172.17.0.3:0 2024-11-09T17:28:51,946 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegionServer(1224): stopping server c28fd47d982f,44083,1731173137386 2024-11-09T17:28:51,946 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegionServer(1250): stopping server c28fd47d982f,44083,1731173137386; all regions closed. 2024-11-09T17:28:51,946 DEBUG [M:0;c28fd47d982f:44083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T17:28:51,946 DEBUG [M:0;c28fd47d982f:44083 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T17:28:51,947 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T17:28:51,947 DEBUG [M:0;c28fd47d982f:44083 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T17:28:51,947 DEBUG [master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.large.0-1731173140816 {}] cleaner.HFileCleaner(306): Exit Thread[master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.large.0-1731173140816,5,FailOnTimeoutGroup] 2024-11-09T17:28:51,947 DEBUG [master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.small.0-1731173140816 {}] cleaner.HFileCleaner(306): Exit Thread[master/c28fd47d982f:0:becomeActiveMaster-HFileCleaner.small.0-1731173140816,5,FailOnTimeoutGroup] 2024-11-09T17:28:51,948 INFO [M:0;c28fd47d982f:44083 {}] hbase.ChoreService(370): Chore service for: master/c28fd47d982f:0 had [] on shutdown 2024-11-09T17:28:51,948 DEBUG [M:0;c28fd47d982f:44083 {}] master.HMaster(1733): Stopping service threads 2024-11-09T17:28:51,948 INFO [M:0;c28fd47d982f:44083 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T17:28:51,949 ERROR [M:0;c28fd47d982f:44083 {}] procedure2.ProcedureExecutor(722): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (59733779) connection to localhost/127.0.0.1:46491 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:46491,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T17:28:51,950 INFO [M:0;c28fd47d982f:44083 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T17:28:51,951 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T17:28:51,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T17:28:51,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T17:28:51,958 DEBUG [M:0;c28fd47d982f:44083 {}] zookeeper.ZKUtil(347): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T17:28:51,958 WARN [M:0;c28fd47d982f:44083 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T17:28:51,958 INFO [M:0;c28fd47d982f:44083 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-09T17:28:51,958 INFO [M:0;c28fd47d982f:44083 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T17:28:51,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T17:28:51,958 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T17:28:51,958 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:28:51,958 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:28:51,958 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T17:28:51,958 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:28:51,958 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=728.08 KB heapSize=892.86 KB 2024-11-09T17:28:51,972 DEBUG [M:0;c28fd47d982f:44083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/723a1f1f517447b2b09f3087bb0812bb is 82, key is hbase:meta,,1/info:regioninfo/1731173142251/Put/seqid=0 2024-11-09T17:28:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742380_1556 (size=5672) 2024-11-09T17:28:52,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T17:28:52,037 INFO [RS:0;c28fd47d982f:45633 {}] regionserver.HRegionServer(1307): Exiting; stopping=c28fd47d982f,45633,1731173138129; zookeeper connection closed. 2024-11-09T17:28:52,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45633-0x10121eb5b6a0001, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T17:28:52,038 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47c15e5b {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47c15e5b 2024-11-09T17:28:52,039 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T17:28:52,376 INFO [M:0;c28fd47d982f:44083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/723a1f1f517447b2b09f3087bb0812bb 2024-11-09T17:28:52,406 DEBUG [M:0;c28fd47d982f:44083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0afed85601f543e4bc4b8bbd52727af9 is 2278, key is \x00\x00\x00\x00\x00\x00\x00a/proc:d/1731173235799/Put/seqid=0 2024-11-09T17:28:52,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742381_1557 (size=42291) 2024-11-09T17:28:52,811 INFO [M:0;c28fd47d982f:44083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=727.53 KB at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0afed85601f543e4bc4b8bbd52727af9 2024-11-09T17:28:52,817 INFO [M:0;c28fd47d982f:44083 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0afed85601f543e4bc4b8bbd52727af9 2024-11-09T17:28:52,835 DEBUG [M:0;c28fd47d982f:44083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6209f360e124283844a08c4c58ec69e is 69, key is c28fd47d982f,45633,1731173138129/rs:state/1731173140875/Put/seqid=0 2024-11-09T17:28:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073742382_1558 (size=5156) 2024-11-09T17:28:53,239 INFO [M:0;c28fd47d982f:44083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2027 (bloomFilter=true), to=hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6209f360e124283844a08c4c58ec69e 2024-11-09T17:28:53,262 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/723a1f1f517447b2b09f3087bb0812bb as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/723a1f1f517447b2b09f3087bb0812bb 2024-11-09T17:28:53,269 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/723a1f1f517447b2b09f3087bb0812bb, entries=8, sequenceid=2027, filesize=5.5 K 2024-11-09T17:28:53,271 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0afed85601f543e4bc4b8bbd52727af9 as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0afed85601f543e4bc4b8bbd52727af9 2024-11-09T17:28:53,276 INFO [M:0;c28fd47d982f:44083 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0afed85601f543e4bc4b8bbd52727af9 2024-11-09T17:28:53,276 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0afed85601f543e4bc4b8bbd52727af9, entries=163, sequenceid=2027, filesize=41.3 K 2024-11-09T17:28:53,277 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a6209f360e124283844a08c4c58ec69e as hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6209f360e124283844a08c4c58ec69e 2024-11-09T17:28:53,282 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:46491/user/jenkins/test-data/c25aac29-fac7-4710-528d-ae3f75b5d3f7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a6209f360e124283844a08c4c58ec69e, entries=1, sequenceid=2027, filesize=5.0 K 2024-11-09T17:28:53,283 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(3040): Finished flush of dataSize ~728.08 KB/745557, heapSize ~892.56 KB/913984, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1325ms, sequenceid=2027, compaction requested=false 2024-11-09T17:28:53,285 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T17:28:53,285 DEBUG [M:0;c28fd47d982f:44083 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-09T17:28:53,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35365 is added to blk_1073741830_1006 (size=878487) 2024-11-09T17:28:53,289 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-09T17:28:53,289 INFO [M:0;c28fd47d982f:44083 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-09T17:28:53,289 INFO [M:0;c28fd47d982f:44083 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.3:44083 2024-11-09T17:28:53,300 DEBUG [M:0;c28fd47d982f:44083 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/c28fd47d982f,44083,1731173137386 already deleted, retry=false 2024-11-09T17:28:53,409 INFO [M:0;c28fd47d982f:44083 {}] regionserver.HRegionServer(1307): Exiting; stopping=c28fd47d982f,44083,1731173137386; zookeeper connection closed. 2024-11-09T17:28:53,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T17:28:53,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44083-0x10121eb5b6a0000, quorum=127.0.0.1:57222, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T17:28:53,419 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1f79ec76{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T17:28:53,421 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@576ebda6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T17:28:53,422 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T17:28:53,422 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4727fac8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T17:28:53,422 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47db50b9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.log.dir/,STOPPED} 2024-11-09T17:28:53,425 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T17:28:53,425 WARN [BP-237782476-172.17.0.3-1731173134238 heartbeating to localhost/127.0.0.1:46491 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T17:28:53,425 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T17:28:53,425 WARN [BP-237782476-172.17.0.3-1731173134238 heartbeating to localhost/127.0.0.1:46491 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-237782476-172.17.0.3-1731173134238 (Datanode Uuid b69a8dc7-51eb-4008-863f-c33ebf910a7f) service to localhost/127.0.0.1:46491 2024-11-09T17:28:53,427 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data1/current/BP-237782476-172.17.0.3-1731173134238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T17:28:53,428 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/cluster_4af920cc-f1a4-0de0-1c35-047715a57a6e/dfs/data/data2/current/BP-237782476-172.17.0.3-1731173134238 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T17:28:53,428 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T17:28:53,434 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@b03fcff{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T17:28:53,434 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@e0a3ea0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T17:28:53,434 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T17:28:53,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62a9beb2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T17:28:53,435 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@8167a4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/c0f919a2-e8ca-4ff3-0eaa-01af9177f156/hadoop.log.dir/,STOPPED} 2024-11-09T17:28:53,448 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-09T17:28:53,558 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down