CI Summary

Failed: Timeout >900.0s-self = 

    @pytest.mark.resource_intensive
    def test_network_topology_strategy_users(self):
        """
        Test for multiple datacenters, users table.
        """
        self.nodes = [3, 3]
        self.rf = OrderedDict([('dc1', 3), ('dc2', 3)])
    
        combinations = [
            (ConsistencyLevel.ALL, ConsistencyLevel.ALL),
            (ConsistencyLevel.QUORUM, ConsistencyLevel.QUORUM),
            (ConsistencyLevel.EACH_QUORUM, ConsistencyLevel.LOCAL_QUORUM),
            (ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.LOCAL_QUORUM),
            (ConsistencyLevel.ALL, ConsistencyLevel.ONE),
            (ConsistencyLevel.ONE, ConsistencyLevel.ALL),
            (ConsistencyLevel.QUORUM, ConsistencyLevel.ONE),
            (ConsistencyLevel.ONE, ConsistencyLevel.QUORUM),
            (ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.ONE),
            (ConsistencyLevel.ONE, ConsistencyLevel.LOCAL_QUORUM),
            (ConsistencyLevel.EACH_QUORUM, ConsistencyLevel.ONE),
            (ConsistencyLevel.TWO, ConsistencyLevel.TWO),
            (ConsistencyLevel.ONE, ConsistencyLevel.THREE),
            (ConsistencyLevel.THREE, ConsistencyLevel.ONE),
            (ConsistencyLevel.ANY, ConsistencyLevel.ONE),
            (ConsistencyLevel.ONE, ConsistencyLevel.ONE),
            (ConsistencyLevel.ONE, ConsistencyLevel.TWO),
            (ConsistencyLevel.TWO, ConsistencyLevel.ONE),
            (ConsistencyLevel.QUORUM, ConsistencyLevel.SERIAL, ConsistencyLevel.SERIAL),
            (ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.LOCAL_SERIAL),
            (ConsistencyLevel.QUORUM, ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL),
            (ConsistencyLevel.LOCAL_QUORUM, ConsistencyLevel.SERIAL, ConsistencyLevel.LOCAL_SERIAL),
        ]
    
        logger.debug("Testing multiple dcs, users")
>       self._run_test_function_in_parallel(TestAccuracy.Validation.validate_users, self.nodes, list(self.rf.values()), combinations),

consistency_test.py:651: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 
valid_fcn = 
nodes = [3, 3], rf_factors = [3, 3]
combinations = [(5, 5), (4, 4), (7, 6), (6, 6), (5, 1), (1, 5), ...]

    def _run_test_function_in_parallel(self, valid_fcn, nodes, rf_factors, combinations):
        """
        Run a test function in parallel.
        """
    
        requires_local_reads = False
        for combination in combinations:
            for i, _ in enumerate(nodes):
                expected_consistency = self.get_expected_consistency(i, rf_factors, combination[0], combination[1])
                if not expected_consistency.is_strong:
                    # if at least one combination does not reach strong consistency, in order to validate weak
                    # consistency we require local reads, see CASSANDRA-12092 for details.
                    requires_local_reads = True
                    break
    
            if requires_local_reads:
                break
    
        self._start_cluster(save_sessions=True, requires_local_reads=requires_local_reads)
    
        input_queue = queue.Queue()
        exceptions_queue = queue.Queue()
    
        def run():
            while not input_queue.empty():
                try:
                    v = TestAccuracy.Validation(self, self.sessions, nodes, rf_factors, *input_queue.get(block=False))
                    valid_fcn(v)
                except queue.Empty:
                    pass
                except Exception:
                    exceptions_queue.put(sys.exc_info())
    
        start = 0
        num_keys = 50
        for combination in combinations:
            input_queue.put((start, start + num_keys) + combination)
            start += num_keys
    
        threads = []
        for n in range(0, 8):
            t = threading.Thread(target=run)
            t.setDaemon(True)
            t.start()
            threads.append(t)
    
        logger.debug("Waiting for workers to complete")
        while exceptions_queue.empty():
>           time.sleep(0.1)
E           Failed: Timeout >900.0s

consistency_test.py:559: Failed

ccmlib.node.NodeError: C* process with 9816 is terminated-self = 

    @pytest.mark.resource_intensive
    def test_replace_shutdown_node(self):
        """
        @jira_ticket CASSANDRA-9871
        Test that we can replace a node that is shutdown gracefully.
        """
>       self._test_replace_node(gently=True)

replace_address_test.py:289: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
replace_address_test.py:313: in _test_replace_node
    self._do_replace(same_address=same_address)
replace_address_test.py:122: in _do_replace
    self.replacement_node.start(jvm_args=extra_jvm_args,
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:928: in start
    self.wait_for_binary_interface(from_mark=self.mark)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:711: in wait_for_binary_interface
    self.watch_log_for("Starting listening for CQL clients", **kwargs)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:608: in watch_log_for
    self.raise_node_error_if_cassandra_process_is_terminated()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def raise_node_error_if_cassandra_process_is_terminated(self):
        if not self._is_pid_running():
            msg = "C* process with {pid} is terminated".format(pid=self.pid)
            common.debug(msg)
>           raise NodeError(msg)
E           ccmlib.node.NodeError: C* process with 9816 is terminated

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:692: NodeError

Failed: Timeout >900.0s-self = 

    @since('3.10')
    def test_disk_balance_after_joining_ring_stcs(self):
        """
        @jira_ticket CASSANDRA-13948
        """
>       self._disk_balance_after_joining_ring_test(lcs=False)

disk_balance_test.py:271: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
disk_balance_test.py:331: in _disk_balance_after_joining_ring_test
    self._assert_balanced_after_boundary_change(node1, total_keys, lcs)
disk_balance_test.py:353: in _assert_balanced_after_boundary_change
    node.stress(['read', 'n={}'.format(total_keys), "no-warmup", "cl=ALL", "-pop", "seq=1...{}".format(total_keys), "-rate", "threads=1"])
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1529: in stress
    return handle_external_tool_process(p, ['stress'] + stress_options)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:2325: in handle_external_tool_process
    out, err = process.communicate()
/usr/lib/python3.8/subprocess.py:1028: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib/python3.8/subprocess.py:1868: in _communicate
    ready = selector.select(timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = , timeout = None

    def select(self, timeout=None):
        # This is shared between poll() and epoll().
        # epoll() has a different signature and handling of timeout parameter.
        if timeout is None:
            timeout = None
        elif timeout <= 0:
            timeout = 0
        else:
            # poll() has a resolution of 1 millisecond, round away from
            # zero to wait *at least* timeout seconds.
            timeout = math.ceil(timeout * 1e3)
        ready = []
        try:
>           fd_event_list = self._selector.poll(timeout)
E           Failed: Timeout >900.0s

/usr/lib/python3.8/selectors.py:415: Failed

cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042-self = 

    def test_bulk_round_trip_non_prepared_statements(self):
        """
        Test bulk import with default stress import (one row per operation) and without
        prepared statements.
    
        @jira_ticket CASSANDRA-11053
        """
>       self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=100000,
                                   copy_from_options={'PREPAREDSTATEMENTS': False})

cqlsh_tests/test_cqlsh_copy.py:2475: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042-self = 

    def test_bulk_round_trip_blogposts_with_max_connections(self):
        """
        Same as test_bulk_round_trip_blogposts but limit the maximum number of concurrent connections a host will
        accept to simulate a failed connection to a replica that is up. Here we are interested in testing COPY TO,
        where we should have at most worker_processes * nodes connections + 1 connections, the +1 is the cqlsh
        connection. For COPY FROM the driver handles retries, we use only 2 worker processes to make sure it succeeds.
    
        @jira_ticket CASSANDRA-10938
        """
>       self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=10000,
                                   configuration_options={'native_transport_max_concurrent_connections': '12',
                                                          'batch_size_warn_threshold_in_kb': '10'},
                                   profile=os.path.join(os.path.dirname(os.path.realpath(__file__)), 'blogposts.yaml'),
                                   stress_table='stresscql.blogposts',
                                   copy_to_options={'NUMPROCESSES': 5, 'MAXATTEMPTS': 20},
                                   copy_from_options={'NUMPROCESSES': 2})

cqlsh_tests/test_cqlsh_copy.py:2498: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

Failed: Timeout >900.0s-self = 

    def test_dc_repair(self):
        """
        * Set up a multi DC cluster
        * Perform a -dc repair on two dc's
        * Assert only nodes on those dcs were repaired
        """
        cluster = self._setup_multi_dc()
        node1 = cluster.nodes["node1"]
        node2 = cluster.nodes["node2"]
        node3 = cluster.nodes["node3"]
    
        logger.debug("starting repair...")
        opts = ["-dc", "dc1", "-dc", "dc2"]
        opts += _repair_options(self.cluster.version(), ks="ks")
>       node1.repair(opts)

repair_tests/repair_test.py:660: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1576: in repair
    return self.nodetool(cmd)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1032: in nodetool
    return handle_external_tool_process(p, ['nodetool', '-h', 'localhost', '-p', str(self.jmx_port)] + shlex.split(cmd))
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:2325: in handle_external_tool_process
    out, err = process.communicate()
/usr/lib/python3.8/subprocess.py:1028: in communicate
    stdout, stderr = self._communicate(input, endtime, timeout)
/usr/lib/python3.8/subprocess.py:1868: in _communicate
    ready = selector.select(timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = , timeout = None

    def select(self, timeout=None):
        # This is shared between poll() and epoll().
        # epoll() has a different signature and handling of timeout parameter.
        if timeout is None:
            timeout = None
        elif timeout <= 0:
            timeout = 0
        else:
            # poll() has a resolution of 1 millisecond, round away from
            # zero to wait *at least* timeout seconds.
            timeout = math.ceil(timeout * 1e3)
        ready = []
        try:
>           fd_event_list = self._selector.poll(timeout)
E           Failed: Timeout >900.0s

/usr/lib/python3.8/selectors.py:415: Failed

failed on teardown with "Unexpected error found in node logs (see stdout for full details). Errors: [[node1] 'ERROR [OptionalTasks:1] 2024-11-21 10:03:35,417 JVMStabilityInspector.java:70 - Exception in thread Thread[OptionalTasks:1,5,OptionalTasks]\norg.apache.cassandra.exceptions.WriteTimeoutException: Operation timed out - received only 0 responses.\n\tat org.apache.cassandra.service.AbstractWriteResponseHandler.throwTimeout(AbstractWriteResponseHandler.java:149)\n\tat org.apache.cassandra.service.AbstractWriteResponseHandler.get(AbstractWriteResponseHandler.java:127)\n\tat org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:905)\n\tat org.apache.cassandra.service.StorageProxy.mutateWithTriggers(StorageProxy.java:1157)\n\tat org.apache.cassandra.cql3.statements.ModificationStatement.executeWithoutCondition(ModificationStatement.java:529)\n\tat org.apache.cassandra.cql3.statements.ModificationStatement.execute(ModificationStatement.java:502)\n\tat org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:259)\n\tat org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:352)\n\tat org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:370)\n\tat org.apache.cassandra.schema.SystemDistributedKeyspace.startViewBuild(SystemDistributedKeyspace.java:327)\n\tat org.apache.cassandra.db.view.ViewBuilder.start(ViewBuilder.java:94)\n\tat org.apache.cassandra.db.view.View.build(View.java:214)\n\tat org.apache.cassandra.db.view.ViewManager.buildAllViews(ViewManager.java:182)\n\tat org.apache.cassandra.service.CassandraDaemon.lambda$setup$2(CassandraDaemon.java:396)\n\tat org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)']"-Unexpected error found in node logs (see stdout for full details). Errors: [[node1] 'ERROR [OptionalTasks:1] 2024-11-21 10:03:35,417 JVMStabilityInspector.java:70 - Exception in thread Thread[OptionalTasks:1,5,OptionalTasks]\norg.apache.cassandra.exceptions.WriteTimeoutException: Operation timed out - received only 0 responses.\n\tat org.apache.cassandra.service.AbstractWriteResponseHandler.throwTimeout(AbstractWriteResponseHandler.java:149)\n\tat org.apache.cassandra.service.AbstractWriteResponseHandler.get(AbstractWriteResponseHandler.java:127)\n\tat org.apache.cassandra.service.StorageProxy.mutate(StorageProxy.java:905)\n\tat org.apache.cassandra.service.StorageProxy.mutateWithTriggers(StorageProxy.java:1157)\n\tat org.apache.cassandra.cql3.statements.ModificationStatement.executeWithoutCondition(ModificationStatement.java:529)\n\tat org.apache.cassandra.cql3.statements.ModificationStatement.execute(ModificationStatement.java:502)\n\tat org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:259)\n\tat org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:352)\n\tat org.apache.cassandra.cql3.QueryProcessor.process(QueryProcessor.java:370)\n\tat org.apache.cassandra.schema.SystemDistributedKeyspace.startViewBuild(SystemDistributedKeyspace.java:327)\n\tat org.apache.cassandra.db.view.ViewBuilder.start(ViewBuilder.java:94)\n\tat org.apache.cassandra.db.view.View.build(View.java:214)\n\tat org.apache.cassandra.db.view.ViewManager.buildAllViews(ViewManager.java:182)\n\tat org.apache.cassandra.service.CassandraDaemon.lambda$setup$2(CassandraDaemon.java:396)\n\tat org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)\n\tat java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)\n\tat java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)\n\tat java.base/java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)\n\tat java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)']

cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042-self = 

    @since('3.0.5')
    def test_bulk_round_trip_with_backoff(self):
        """
        Test bulk import with default stress import (one row per operation) and COPY options
        that exercise the new back-off policy introduced by CASSANDRA-11320.
    
        @jira_ticket CASSANDRA-11320
        """
>       self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=250000,
                                   copy_from_options={'MAXINFLIGHTMESSAGES': 64, 'MAXPENDINGCHUNKS': 1})

cqlsh_tests/test_cqlsh_copy.py:2553: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.3:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.3:9042-self = 

    def test_bulk_round_trip_default(self):
        """
        Test bulk import with default stress import (one row per operation)
    
        @jira_ticket CASSANDRA-9302
        """
>       self._test_bulk_round_trip(nodes=3, partitioner="murmur3", num_operations=100000)

cqlsh_tests/test_cqlsh_copy.py:2466: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cqlsh_tests/test_cqlsh_copy.py:2436: in _test_bulk_round_trip
    num_records = create_records()
cqlsh_tests/test_cqlsh_copy.py:2409: in create_records
    ret = rows_to_list(self.session.execute(count_statement))[0][0]
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.3:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.3:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

Failed: Timeout >900.0s-self = 

    def test_local_dc_repair(self):
        """
        * Set up a multi DC cluster
        * Perform a -local repair on one DC
        * Assert only nodes in that DC are repaired
        """
        cluster = self._setup_multi_dc()
        node1 = cluster.nodes["node1"]
        node2 = cluster.nodes["node2"]
    
        logger.debug("starting repair...")
        opts = ["-local"]
        opts += _repair_options(self.cluster.version(), ks="ks")
>       node1.repair(opts)

repair_tests/repair_test.py:631: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1576: in repair
    return self.nodetool(cmd)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1031: in nodetool
    p = self.nodetool_process(cmd)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1023: in nodetool_process
    env = self.get_env()
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:241: in get_env
    env = common.update_java_version(jvm_version=None,
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/common.py:962: in update_java_version
    current_java_version = get_jdk_version_int(env=env)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/common.py:828: in get_jdk_version_int
    jdk_version_str = get_jdk_version(process, env)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/common.py:844: in get_jdk_version
    version = subprocess.check_output([process, '-version'], stderr=subprocess.STDOUT, env=env)
/usr/lib/python3.8/subprocess.py:415: in check_output
    return run(*popenargs, stdout=PIPE, timeout=timeout, check=True,
/usr/lib/python3.8/subprocess.py:495: in run
    stdout, stderr = process.communicate(input, timeout=timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = , input = None, timeout = None

    def communicate(self, input=None, timeout=None):
        """Interact with process: Send data to stdin and close it.
        Read data from stdout and stderr, until end-of-file is
        reached.  Wait for process to terminate.
    
        The optional "input" argument should be data to be sent to the
        child process, or None, if no data should be sent to the child.
        communicate() returns a tuple (stdout, stderr).
    
        By default, all communication is in bytes, and therefore any
        "input" should be bytes, and the (stdout, stderr) will be bytes.
        If in text mode (indicated by self.text_mode), any "input" should
        be a string, and (stdout, stderr) will be strings decoded
        according to locale encoding, or by "encoding" if set. Text mode
        is triggered by setting any of text, encoding, errors or
        universal_newlines.
        """
    
        if self._communication_started and input:
            raise ValueError("Cannot send input after starting communication")
    
        # Optimization: If we are not worried about timeouts, we haven't
        # started communicating, and we have one or zero pipes, using select()
        # or threads is unnecessary.
        if (timeout is None and not self._communication_started and
            [self.stdin, self.stdout, self.stderr].count(None) >= 2):
            stdout = None
            stderr = None
            if self.stdin:
                self._stdin_write(input)
            elif self.stdout:
>               stdout = self.stdout.read()
E               Failed: Timeout >900.0s

/usr/lib/python3.8/subprocess.py:1015: Failed

Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.-junit.framework.AssertionFailedError: Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor49.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.apache.cassandra.anttasks.TestHelper.execute(TestHelper.java:53)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Exception found expected null, but was:-junit.framework.AssertionFailedError: Exception found expected null, but was:
	at org.apache.cassandra.distributed.test.DistributedRepairUtils.lambda$assertParentRepairSuccess$4(DistributedRepairUtils.java:135)
	at org.apache.cassandra.distributed.test.DistributedRepairUtils.validateExistingParentRepair(DistributedRepairUtils.java:170)
	at org.apache.cassandra.distributed.test.DistributedRepairUtils.assertParentRepairSuccess(DistributedRepairUtils.java:130)
	at org.apache.cassandra.distributed.test.RepairTest.testForcedNormalRepairWithOneNodeDown(RepairTest.java:211)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Uncaught exceptions were thrown during test-org.apache.cassandra.distributed.shared.ShutdownException: Uncaught exceptions were thrown during test
	at org.apache.cassandra.distributed.impl.AbstractCluster.checkAndResetUncaughtExceptions(AbstractCluster.java:1117)
	at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1103)
	at org.apache.cassandra.distributed.test.RepairTest.closeCluster(RepairTest.java:160)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	Suppressed: java.lang.IllegalStateException: complete already: (failure: java.lang.RuntimeException: Did not get replies from all endpoints.)
		at org.apache.cassandra.utils.concurrent.AsyncPromise.setSuccess(AsyncPromise.java:106)
		at org.apache.cassandra.service.ActiveRepairService$2.ack(ActiveRepairService.java:736)
		at org.apache.cassandra.service.ActiveRepairService$2.onResponse(ActiveRepairService.java:712)
		at org.apache.cassandra.repair.messages.RepairMessage$2.onResponse(RepairMessage.java:187)
		at org.apache.cassandra.net.ResponseVerbHandler.doVerb(ResponseVerbHandler.java:58)
		at org.apache.cassandra.net.InboundSink.lambda$new$0(InboundSink.java:78)
		at org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:64)
		at org.apache.cassandra.net.InboundSink$Filtered.accept(InboundSink.java:50)
		at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:97)
		at org.apache.cassandra.net.InboundSink.accept(InboundSink.java:45)
		at org.apache.cassandra.net.InboundMessageHandler$ProcessMessage.run(InboundMessageHandler.java:430)
		at org.apache.cassandra.concurrent.ExecutionFailure$1.run(ExecutionFailure.java:133)
		at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:143)
		at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
		at java.base/java.lang.Thread.run(Thread.java:840)

Error in test '4.1.8 -> [5.0.3]' while upgrading to '5.0.3'; successful upgrades [4.0.16 -> [5.0.3]]-junit.framework.AssertionFailedError: Error in test '4.1.8 -> [5.0.3]' while upgrading to '5.0.3'; successful upgrades [4.0.16 -> [5.0.3]]
	at org.apache.cassandra.distributed.upgrade.UpgradeTestBase$TestCase.run(UpgradeTestBase.java:369)
	at org.apache.cassandra.distributed.upgrade.MixedModeAvailabilityTestBase.testAvailability(MixedModeAvailabilityTestBase.java:128)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.lang.AssertionError: Unexpected error while reading in case write-read consistency QUORUM-QUORUM with upgraded coordinator and 1 nodes down: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 2 responses.
	at org.apache.cassandra.distributed.upgrade.MixedModeAvailabilityTestBase.lambda$testAvailability$3(MixedModeAvailabilityTestBase.java:116)
	at java.base/java.util.Map.forEach(Map.java:661)
	at org.apache.cassandra.distributed.upgrade.MixedModeAvailabilityTestBase.lambda$testAvailability$4(MixedModeAvailabilityTestBase.java:88)
	at org.apache.cassandra.distributed.upgrade.UpgradeTestBase$TestCase.run(UpgradeTestBase.java:360)
Caused by: org.apache.cassandra.exceptions.ReadTimeoutException: Operation timed out - received only 2 responses.
	at org.apache.cassandra.service.reads.ReadCallback.awaitResults(ReadCallback.java:178)
	at org.apache.cassandra.service.reads.AbstractReadExecutor.awaitResponses(AbstractReadExecutor.java:412)
	at org.apache.cassandra.service.StorageProxy.fetchRows(StorageProxy.java:2147)
	at org.apache.cassandra.service.StorageProxy.readRegular(StorageProxy.java:2010)
	at org.apache.cassandra.service.StorageProxy.read(StorageProxy.java:1884)
	at org.apache.cassandra.db.SinglePartitionReadCommand$Group.execute(SinglePartitionReadCommand.java:1285)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:427)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:340)
	at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:108)
	at org.apache.cassandra.distributed.impl.CoordinatorHelper.unsafeExecuteInternal(CoordinatorHelper.java:67)
	at org.apache.cassandra.distributed.impl.CoordinatorHelper.unsafeExecuteInternal(CoordinatorHelper.java:47)
	at org.apache.cassandra.distributed.impl.Coordinator.unsafeExecuteInternal(Coordinator.java:94)
	at org.apache.cassandra.distributed.impl.Coordinator.lambda$executeWithResult$0(Coordinator.java:61)
	at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)