CI Summary

ccmlib.node.TimeoutError: 07 May 2024 04:58:35 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:57:35,898 M
 Tail: ...OptionalTasks:1] 2024-05-07 04:57:56,649 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715057855.8966608, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:56,649 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 04:58:35 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:57:35,898 M
E            Tail: ...OptionalTasks:1] 2024-05-07 04:57:56,649 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 05:23:02 [node2] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:22:02,584 M
 Tail: ... 2024-05-07 05:22:38,315 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715059322.5997725, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:23:02 [node2] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:22:02,584 M
E            Tail: ... 2024-05-07 05:22:38,315 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

AssertionError: Commitlog was written with durable writes disabled
assert 90112 == 86016
  +90112
  -86016-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 90112 == 86016
E         +90112
E         -86016

configuration_test.py:104: AssertionError

cassandra.WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received 0/1 responses." info={'consistency': 'ONE', 'required_responses': 1, 'received_responses': 0, 'write_type': 'COUNTER'}-self = 

    def test_counter_node_down(self):
        """
        @jira_ticket CASSANDRA-17411
        """
        cluster = self.cluster
        cluster.populate(3).start()
        node1, node2, node3 = cluster.nodelist()
        session = self.patient_cql_connection(node1)
        create_ks(session, 'counter_tests', 3)
    
        session.execute("""
            CREATE TABLE IF NOT EXISTS counter_cs (
                key text PRIMARY KEY,
                count counter
            )
            """)
    
        node2.stop(gently=False, wait_other_notice=True)
    
        for _ in range(0, 20):
>           session.execute("UPDATE counter_cs SET count = count + 1 WHERE key = 'test'")

counter_test.py:447: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received 0/1 responses." info={'consistency': 'ONE', 'required_responses': 1, 'received_responses': 0, 'write_type': 'COUNTER'}

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: WriteTimeout

ccmlib.node.TimeoutError: 07 May 2024 05:07:10 [node1] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:06:10,641 M
 Tail: ...OptionalTasks:1] 2024-05-07 05:06:36,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715058370.6484132, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:36,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:07:10 [node1] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:06:10,641 M
E            Tail: ...OptionalTasks:1] 2024-05-07 05:06:36,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 05:14:49 [node2] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: INFO  [OptionalTasks:1] 2024-05-07 05:13:49,165 Ca
 Tail: ... 2024-05-07 05:14:25,782 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715058829.195756, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: INFO  [OptionalTa...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:14:49 [node2] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: INFO  [OptionalTasks:1] 2024-05-07 05:13:49,165 Ca
E            Tail: ... 2024-05-07 05:14:25,782 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

AssertionError: Commitlog was written with durable writes disabled
assert 90112 == 86016
  +90112
  -86016-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 90112 == 86016
E         +90112
E         -86016

configuration_test.py:104: AssertionError

ccmlib.node.TimeoutError: 07 May 2024 05:55:57 [node3] after 20.02/20 seconds Missing: ['Use cassandra.replace_address if you want to replace this node'] not found in system.log:
 Head: INFO  [main] 2024-05-07 05:55:38,565 YamlConfigura
 Tail: ...1143 - InetAddress /127.0.0.1:7000 is now UP
INFO  [GossipStage:1] 2024-05-07 05:55:49,322 Gossiper.java:1143 - InetAddress /127.0.0.1:7000 is now UP-self = 

    @since('3.6')
    def test_fail_without_replace(self):
        """
        When starting a node from a clean slate with the same address as
        an existing down node, the node should error out even when
        auto_bootstrap = false (or the node is a seed) and tell the user
        to use replace_address.
        @jira_ticket CASSANDRA-10134
        """
        self.fixture_dtest_setup.ignore_log_patterns = list(self.fixture_dtest_setup.ignore_log_patterns) + [
            r'Exception encountered during startup']
    
        self._setup(n=3)
        self._insert_data()
        node1, node2, node3 = self.cluster.nodelist()
    
        mark = None
        for auto_bootstrap in (True, False):
            logger.debug("Stopping node 3.")
            node3.stop(gently=False)
    
            # completely delete the data, commitlog, and saved caches
            for d in chain([os.path.join(node3.get_path(), "commitlogs")],
                           [os.path.join(node3.get_path(), "saved_caches")],
                           node3.data_directories()):
                if os.path.exists(d):
                    rmtree(d)
    
            node3.set_configuration_options(values={'auto_bootstrap': auto_bootstrap})
            logger.debug("Starting node 3 with auto_bootstrap = {val}".format(val=auto_bootstrap))
            node3.start(wait_other_notice=False)
>           node3.watch_log_for('Use cassandra.replace_address if you want to replace this node', from_mark=mark, timeout=20)

replace_address_test.py:395: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715061337.2529073, timeout = 20
msg = "Missing: ['Use cassandra.replace_address if you want to replace this node'] not found in system.log:\n Head: INFO  [m... is now UP\nINFO  [GossipStage:1] 2024-05-07 05:55:49,322 Gossiper.java:1143 - InetAddress /127.0.0.1:7000 is now UP\n"
node = 'node3'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:55:57 [node3] after 20.02/20 seconds Missing: ['Use cassandra.replace_address if you want to replace this node'] not found in system.log:
E            Head: INFO  [main] 2024-05-07 05:55:38,565 YamlConfigura
E            Tail: ...1143 - InetAddress /127.0.0.1:7000 is now UP
E           INFO  [GossipStage:1] 2024-05-07 05:55:49,322 Gossiper.java:1143 - InetAddress /127.0.0.1:7000 is now UP

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 06:06:50 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:05:50,894 M
 Tail: ...OptionalTasks:1] 2024-05-07 06:06:12,594 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715061950.8940258, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:12,594 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 06:06:50 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:05:50,894 M
E            Tail: ...OptionalTasks:1] 2024-05-07 06:06:12,594 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 06:21:21 [node2] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:20:21,147 M
 Tail: ... 2024-05-07 06:20:52,296 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715062821.1482842, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 06:21:21 [node2] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:20:21,147 M
E            Tail: ... 2024-05-07 06:20:52,296 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 06:06:20 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:05:20,653 M
 Tail: ...OptionalTasks:1] 2024-05-07 06:05:43,082 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715061920.6571033, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:43,082 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 06:06:20 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:05:20,653 M
E            Tail: ...OptionalTasks:1] 2024-05-07 06:05:43,082 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 06:22:00 [node2] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:21:00,787 M
 Tail: ... 2024-05-07 06:21:36,939 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715062860.7875738, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 06:22:00 [node2] after 60.07/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 06:21:00,787 M
E            Tail: ... 2024-05-07 06:21:36,939 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 08 May 2024 00:01:58 [node1] after 90.12/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
 Head: 
 Tail: ...-self = 

    @pytest.mark.timeout(3000)
    def test_parallel_upgrade_with_internode_ssl(self):
        """
        Test upgrading cluster all at once (requires cluster downtime), with internode ssl.
        """
>       self.upgrade_scenario(internode_ssl=True)

upgrade_tests/upgrade_through_versions_test.py:402: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
upgrade_tests/upgrade_through_versions_test.py:493: in upgrade_scenario
    self.upgrade_to_version(version_meta, internode_ssl=internode_ssl)
upgrade_tests/upgrade_through_versions_test.py:581: in upgrade_to_version
    node.start(wait_other_notice=400, wait_for_binary_proto=True,
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:922: in start
    self.wait_for_binary_interface(from_mark=self.mark)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:710: in wait_for_binary_interface
    self.watch_log_for("Starting listening for CQL clients", **kwargs)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715126428.241329, timeout = 90
msg = "Missing: ['Starting listening for CQL clients'] not found in system.log:\n Head: \n Tail: ..."
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 08 May 2024 00:01:58 [node1] after 90.12/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
E            Head: 
E            Tail: ...

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 04:57:21 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:56:21,842 M
 Tail: ...OptionalTasks:1] 2024-05-07 04:56:42,807 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715057781.846903, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:42,807 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 04:57:21 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:56:21,842 M
E            Tail: ...OptionalTasks:1] 2024-05-07 04:56:42,807 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 05:02:43 [node2] after 60.05/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:01:43,208 M
 Tail: ... 2024-05-07 05:02:15,925 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715058103.2070642, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:02:43 [node2] after 60.05/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:01:43,208 M
E            Tail: ... 2024-05-07 05:02:15,925 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

cassandra.WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received 0/1 responses." info={'consistency': 'ONE', 'required_responses': 1, 'received_responses': 0, 'write_type': 'COUNTER'}-self = 

    def test_counter_node_down(self):
        """
        @jira_ticket CASSANDRA-17411
        """
        cluster = self.cluster
        cluster.populate(3).start()
        node1, node2, node3 = cluster.nodelist()
        session = self.patient_cql_connection(node1)
        create_ks(session, 'counter_tests', 3)
    
        session.execute("""
            CREATE TABLE IF NOT EXISTS counter_cs (
                key text PRIMARY KEY,
                count counter
            )
            """)
    
        node2.stop(gently=False, wait_other_notice=True)
    
        for _ in range(0, 20):
>           session.execute("UPDATE counter_cs SET count = count + 1 WHERE key = 'test'")

counter_test.py:447: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received 0/1 responses." info={'consistency': 'ONE', 'required_responses': 1, 'received_responses': 0, 'write_type': 'COUNTER'}

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: WriteTimeout

ccmlib.node.TimeoutError: 07 May 2024 04:59:52 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:58:51,963 M
 Tail: ...OptionalTasks:1] 2024-05-07 04:59:14,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup-self = 

    def test_local_query(self):
        """
        Check that a query running locally on the coordinator is reported as slow:
    
        - start a one node cluster with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE and INSERT into a table
        - SELECT * from the table using a retry policy that never retries, and check that the slow
          query log messages are present in the debug logs (we cannot check the logs at info level because
          the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 10,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        # cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
        # introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
        # iteration of non system queries, so that these queries take much longer to complete,
        # see ReadCommand.withStateTracking()
        cluster.populate(1).start(jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                                            "-Dcassandra.test.read_iteration_delay_ms=1"])
        node = cluster.nodelist()[0]
        session = self.patient_cql_connection(node)
    
>       self._assert_logs_slow_queries(node, session)

cql_test.py:1073: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715057931.964778, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...:14,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 04:59:52 [node1] after 60.06/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 04:58:51,963 M
E            Tail: ...OptionalTasks:1] 2024-05-07 04:59:14,550 CassandraDaemon.java:386 - Completed submission of build tasks for any materialized views defined at startup

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 07 May 2024 05:11:35 [node2] after 60.05/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
 Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:10:35,897 M
 Tail: ... 2024-05-07 05:11:10,624 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata-self = 

    def test_remote_query(self):
        """
        Check that a query running on a node other than the coordinator is reported as slow:
    
        - populate the cluster with 2 nodes
        - start one node without having it join the ring
        - start the other one node with slow_query_log_timeout_in_ms set to a small value
          and the read request timeouts set to a large value (to ensure the query is not aborted) and
          read_iteration_delay set to a value big enough for the query to exceed slow_query_log_timeout_in_ms
          (this will cause read queries to take longer than the slow query timeout)
        - CREATE a table
        - INSERT 5000 rows on a session on the node that is not a member of the ring
        - run SELECT statements and check that the slow query messages are present in the debug logs
          (we cannot check the logs at info level because the no spam logger has unpredictable results)
    
        @jira_ticket CASSANDRA-12403
        """
        cluster = self.cluster
        cluster.set_configuration_options(values={'slow_query_log_timeout_in_ms': 1,
                                                  'request_timeout_in_ms': 120000,
                                                  'read_request_timeout_in_ms': 120000,
                                                  'range_request_timeout_in_ms': 120000})
    
        cluster.populate(2)
        node1, node2 = cluster.nodelist()
    
        node1.start(wait_for_binary_proto=True, join_ring=False)  # ensure other node executes queries
        node2.start(wait_for_binary_proto=True,
                    jvm_args=["-Dcassandra.monitoring_report_interval_ms=10",
                              "-Dcassandra.test.read_iteration_delay_ms=2"])  # see above for explanation
    
        session = self.patient_exclusive_cql_connection(node1)
    
>       self._assert_logs_slow_queries(node2, session)

cql_test.py:1108: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_test.py:1160: in _assert_logs_slow_queries
    TestCQLSlowQuery._assert_logs_slow_queries_with_skinny_table(node, session)
cql_test.py:1224: in _assert_logs_slow_queries_with_skinny_table
    TestCQLSlowQuery._assert_logs(node, session, table,
cql_test.py:1522: in _assert_logs
    node.watch_log_for(["operations were slow", logged_query.format(table)],
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1715058635.8938138, timeout = 60
msg = "Missing: ['SELECT \\\\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:\n Head: DEBUG [ScheduledT...ervice.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata\n"
node = 'node2'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 07 May 2024 05:11:35 [node2] after 60.05/60 seconds Missing: ['SELECT \\* FROM ks.t_skinny WHERE k <= 100 AND k > 0'] not found in debug.log:
E            Head: DEBUG [ScheduledTasks:1] 2024-05-07 05:10:35,897 M
E            Tail: ... 2024-05-07 05:11:10,624 StorageService.java:2125 - Ignoring application state LOAD from /127.0.0.1:7000 because it is not a member in token metadata

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

junit.framework.AssertionFailedError
	at org.apache.cassandra.db.commitlog.CommitlogShutdownTest.testShutdownWithPendingTasks(CommitlogShutdownTest.java:97)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$10.evaluate(BMUnitRunner.java:393)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$6.evaluate(BMUnitRunner.java:263)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$1.evaluate(BMUnitRunner.java:97)

expected:<2> but was:<1>-junit.framework.AssertionFailedError: expected:<2> but was:<1>
	at org.apache.cassandra.db.compaction.CompactionsBytemanTest.testSSTableNotEnoughDiskSpaceForCompactionGetsDropped(CompactionsBytemanTest.java:71)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$9.evaluate(BMUnitRunner.java:364)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$6.evaluate(BMUnitRunner.java:263)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$1.evaluate(BMUnitRunner.java:97)

java.lang.OutOfMemoryError: Java heap space: failed reallocation of scalar replaced objects-java.lang.RuntimeException: java.lang.OutOfMemoryError: Java heap space: failed reallocation of scalar replaced objects
	at org.apache.cassandra.distributed.impl.Instance.lambda$startup$7(Instance.java:640)
	at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
	at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.OutOfMemoryError: Java heap space: failed reallocation of scalar replaced objects

All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Channel has been closed))-com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Channel has been closed))
	at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:270)
	at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:109)
	at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
	at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
	at com.datastax.driver.core.Cluster.init(Cluster.java:214)
	at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:387)
	at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:366)
	at com.datastax.driver.core.Cluster.connect(Cluster.java:311)
	at org.apache.cassandra.distributed.test.NativeTransportEncryptionOptionsTest.testOptionalMtlsModeDoNotAllowNonSSLConnections(NativeTransportEncryptionOptionsTest.java:300)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

SHOULD NOT BE ON NODE: 11 -- [(16,30)]: [00, 02, 04, 06, 08, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 42, 44, 46, 48]-junit.framework.AssertionFailedError: SHOULD NOT BE ON NODE: 11 -- [(16,30)]: [00, 02, 04, 06, 08, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 42, 44, 46, 48]
	at org.apache.cassandra.distributed.test.TransientRangeMovementTest.assertAllContained(TransientRangeMovementTest.java:231)
	at org.apache.cassandra.distributed.test.TransientRangeMovement2Test.testMoveForward(TransientRangeMovement2Test.java:143)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Interrupting run because of an exception-java.lang.RuntimeException: Interrupting run because of an exception
	at org.apache.cassandra.harry.runner.Runner.mergeAndThrow(Runner.java:395)
	at org.apache.cassandra.harry.runner.Runner$ConcurrentRunner.runInternal(Runner.java:305)
	at org.apache.cassandra.harry.runner.Runner.run(Runner.java:77)
	at org.apache.cassandra.fuzz.harry.integration.model.ConcurrentQuiescentCheckerIntegrationTest.testConcurrentReadWriteWorkload(ConcurrentQuiescentCheckerIntegrationTest.java:62)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: org.apache.cassandra.exceptions.ReadFailureException: Operation failed - received 0 responses and 1 failures: UNKNOWN from /127.0.0.1:7012
	at org.apache.cassandra.service.reads.ReadCallback.awaitResults(ReadCallback.java:161)
	at org.apache.cassandra.service.reads.AbstractReadExecutor.awaitResponses(AbstractReadExecutor.java:396)
	at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:2071)
	at org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:1941)
	at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1818)
	at org.apache.cassandra.db.SinglePartitionReadCommand.execute(SinglePartitionReadCommand.java:479)
	at org.apache.cassandra.service.pager.AbstractQueryPager.fetchPage(AbstractQueryPager.java:75)
	at org.apache.cassandra.service.pager.SinglePartitionPager.fetchPage(SinglePartitionPager.java:32)
	at org.apache.cassandra.cql3.statements.SelectStatement$Pager$NormalPager.fetchPage(SelectStatement.java:483)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:540)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:347)
	at org.apache.cassandra.distributed.impl.Coordinator$1.hasNext(Coordinator.java:220)
	at org.apache.cassandra.distributed.api.QueryResults$1.hasNext(QueryResults.java:55)
	at org.apache.cassandra.distributed.api.QueryResults$IteratorQueryResult.hasNext(QueryResults.java:166)
	at org.apache.cassandra.distributed.api.QueryResult$1.hasNext(QueryResult.java:76)
	at com.google.common.collect.Iterators.addAll(Iterators.java:365)
	at com.google.common.collect.Lists.newArrayList(Lists.java:146)
	at com.google.common.collect.Iterators.toArray(Iterators.java:349)
	at org.apache.cassandra.harry.sut.injvm.InJvmSutBase.execute(InJvmSutBase.java:156)
	at org.apache.cassandra.harry.sut.injvm.InJvmSutBase.execute(InJvmSutBase.java:139)
	at org.apache.cassandra.harry.sut.SystemUnderTest.executeIdempotent(SystemUnderTest.java:54)
	at org.apache.cassandra.harry.model.SelectHelper.execute(SelectHelper.java:328)
	at org.apache.cassandra.harry.model.SelectHelper.execute(SelectHelper.java:322)
	at org.apache.cassandra.harry.model.QuiescentChecker.lambda$validate$0(QuiescentChecker.java:72)
	at org.apache.cassandra.harry.model.QuiescentChecker.validate(QuiescentChecker.java:78)
	at org.apache.cassandra.harry.model.QuiescentChecker.validate(QuiescentChecker.java:72)
	at org.apache.cassandra.harry.visitors.RandomPartitionValidator.visit(RandomPartitionValidator.java:56)
	at org.apache.cassandra.harry.runner.Runner$ConcurrentRunner.lambda$runInternal$0(Runner.java:296)
	at org.apache.cassandra.harry.runner.Runner.lambda$wrapInterrupt$3(Runner.java:368)
	at org.apache.cassandra.concurrent.InfiniteLoopExecutor.loop(InfiniteLoopExecutor.java:121)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)

Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.-junit.framework.AssertionFailedError: Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Failed on seed 0x37d0ddf3c3fc8fb9-org.apache.cassandra.simulator.SimulationException: Failed on seed 0x37d0ddf3c3fc8fb9
	Suppressed: java.lang.RuntimeException: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.Throwables.maybeFail(Throwables.java:79)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:546)
		at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1098)
		at org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:854)
		at org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:359)
		at org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:344)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner$Run.run(PaxosSimulationRunner.java:34)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
		at org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:101)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	Caused by: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:253)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:538)
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
Caused by: java.lang.RuntimeException: java.lang.ClassCastException: class org.apache.cassandra.dht.Murmur3Partitioner$LongToken cannot be cast to class org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken (org.apache.cassandra.dht.Murmur3Partitioner$LongToken and org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken are in unnamed module of loader org.apache.cassandra.distributed.shared.InstanceClassLoader @679dd234)
	at org.apache.cassandra.simulator.cluster.OnInstanceFlushAndCleanup.lambda$invokableFlushAndCleanup$891cd2d2$1(OnInstanceFlushAndCleanup.java:56)
	at org.apache.cassandra.distributed.api.IInvokableInstance.unsafeRunOnThisThread(IInvokableInstance.java:85)
	at org.apache.cassandra.simulator.systems.SimulatedActionTask.lambda$asSafeRunnable$0(SimulatedActionTask.java:83)
	at org.apache.cassandra.simulator.systems.SimulatedActionTask$1.run(SimulatedActionTask.java:93)
	at org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:318)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)
Caused by: java.lang.ClassCastException: class org.apache.cassandra.dht.Murmur3Partitioner$LongToken cannot be cast to class org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken (org.apache.cassandra.dht.Murmur3Partitioner$LongToken and org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken are in unnamed module of loader org.apache.cassandra.distributed.shared.InstanceClassLoader @679dd234)
	at org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken.compareTo(ReversedLongLocalPartitioner.java:214)
	at org.apache.cassandra.dht.ReversedLongLocalPartitioner$ReversedLongLocalToken.compareTo(ReversedLongLocalPartitioner.java:158)
	at org.apache.cassandra.db.compaction.CompactionManager.needsCleanup(CompactionManager.java:1373)
	at org.apache.cassandra.db.compaction.CompactionManager$5.filterSSTables(CompactionManager.java:656)
	at org.apache.cassandra.db.compaction.CompactionManager.lambda$parallelAllSSTableOperation$0(CompactionManager.java:443)
	at org.apache.cassandra.db.ColumnFamilyStore.withAllSSTables(ColumnFamilyStore.java:2949)
	at org.apache.cassandra.db.compaction.CompactionManager.parallelAllSSTableOperation(CompactionManager.java:434)
	at org.apache.cassandra.db.compaction.CompactionManager.performCleanup(CompactionManager.java:644)
	at org.apache.cassandra.db.ColumnFamilyStore.forceCleanup(ColumnFamilyStore.java:1748)
	at org.apache.cassandra.simulator.cluster.OnInstanceFlushAndCleanup.lambda$invokableFlushAndCleanup$891cd2d2$1(OnInstanceFlushAndCleanup.java:52)