CI Summary

ccmlib.node.TimeoutError: 31 May 2024 22:32:16 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
 Head: INFO  [main] 2024-05-31 22:28:07,576 YamlConfigura
 Tail: ...ava:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)-self = 

    @since('2.2')
    @pytest.mark.resource_intensive
    def test_restart_failed_replace(self):
        """
        Test that if a node fails to replace, it can join the cluster even if the data is wiped.
        """
>       self._test_restart_failed_replace(mode='wipe')

replace_address_test.py:528: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
replace_address_test.py:584: in _test_restart_failed_replace
    self.query_node.watch_log_for("FatClient {} has been silent for 30000ms, removing from gossip".format(node_log_str), timeout=120)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1717194616.0424364, timeout = 120
msg = "Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:\n H...t.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 31 May 2024 22:32:16 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
E            Head: INFO  [main] 2024-05-31 22:28:07,576 YamlConfigura
E            Tail: ...ava:74)
E           	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
E           	at java.base/java.lang.Thread.run(Thread.java:829)

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 31 May 2024 22:33:00 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
 Head: INFO  [main] 2024-05-31 22:28:42,962 YamlConfigura
 Tail: ...ava:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)-self = 

    @since('2.2')
    @pytest.mark.resource_intensive
    def test_restart_failed_replace(self):
        """
        Test that if a node fails to replace, it can join the cluster even if the data is wiped.
        """
>       self._test_restart_failed_replace(mode='wipe')

replace_address_test.py:528: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
replace_address_test.py:584: in _test_restart_failed_replace
    self.query_node.watch_log_for("FatClient {} has been silent for 30000ms, removing from gossip".format(node_log_str), timeout=120)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1717194660.257719, timeout = 120
msg = "Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:\n H...t.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 31 May 2024 22:33:00 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
E            Head: INFO  [main] 2024-05-31 22:28:42,962 YamlConfigura
E            Tail: ...ava:74)
E           	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
E           	at java.base/java.lang.Thread.run(Thread.java:840)

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 31 May 2024 22:17:43 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
 Head: INFO  [main] 2024-05-31 22:13:30,823 YamlConfigura
 Tail: ...ava:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)-self = 

    @since('2.2')
    @pytest.mark.resource_intensive
    def test_restart_failed_replace(self):
        """
        Test that if a node fails to replace, it can join the cluster even if the data is wiped.
        """
>       self._test_restart_failed_replace(mode='wipe')

replace_address_test.py:528: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
replace_address_test.py:584: in _test_restart_failed_replace
    self.query_node.watch_log_for("FatClient {} has been silent for 30000ms, removing from gossip".format(node_log_str), timeout=120)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1717193743.1308017, timeout = 120
msg = "Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:\n H...t.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:829)\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 31 May 2024 22:17:43 [node1] after 120.12/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
E            Head: INFO  [main] 2024-05-31 22:13:30,823 YamlConfigura
E            Tail: ...ava:74)
E           	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
E           	at java.base/java.lang.Thread.run(Thread.java:829)

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

ccmlib.node.TimeoutError: 31 May 2024 22:13:53 [node1] after 120.13/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
 Head: INFO  [main] 2024-05-31 22:09:46,355 YamlConfigura
 Tail: ...ava:74)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)-self = 

    @since('2.2')
    @pytest.mark.resource_intensive
    def test_restart_failed_replace(self):
        """
        Test that if a node fails to replace, it can join the cluster even if the data is wiped.
        """
>       self._test_restart_failed_replace(mode='wipe')

replace_address_test.py:528: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
replace_address_test.py:584: in _test_restart_failed_replace
    self.query_node.watch_log_for("FatClient {} has been silent for 30000ms, removing from gossip".format(node_log_str), timeout=120)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:610: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1717193513.638769, timeout = 120
msg = "Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:\n H...t.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:840)\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 31 May 2024 22:13:53 [node1] after 120.13/120 seconds Missing: ['FatClient /127.0.0.4:7000 has been silent for 30000ms, removing from gossip'] not found in system.log:
E            Head: INFO  [main] 2024-05-31 22:09:46,355 YamlConfigura
E            Tail: ...ava:74)
E           	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
E           	at java.base/java.lang.Thread.run(Thread.java:840)

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

ccmlib.node.ToolError: Subprocess ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']] exited with non-zero status; exit status: 2; 
stdout: Consistency level set to ALL.
TRACING set to ON
; 
stderr: :4:WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received only 2 responses." info={'consistency': 'ALL', 'required_responses': 3, 'received_responses': 2, 'write_type': 'SIMPLE'}-self = 

    @since('3.4')
    def test_tracing_unknown_impl(self):
        """
        Test that Cassandra logs an error, but keeps its default tracing
        behavior, when a nonexistent tracing class is specified.
    
        * set a nonexistent custom tracing class
        * run trace()
        * if running the test on a version with custom tracing classes
          implemented, check that an error about the nonexistent class was
          logged.
    
        @jira_ticket CASSANDRA-10392
        """
        expected_error = 'Cannot use class junk for tracing'
        self.fixture_dtest_setup.ignore_log_patterns = [expected_error]
        session = self.prepare(jvm_args=['-Dcassandra.custom_tracing_class=junk'])
>       self.trace(session)

cql_tracing_test.py:140: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_tracing_test.py:82: in trace
    out, err, _ = node1.run_cqlsh(
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1122: in run_cqlsh
    return handle_external_tool_process(p, ['cqlsh', cmds, cqlsh_options])
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

process = 
cmd_args = ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']]

    def handle_external_tool_process(process, cmd_args):
        out, err = process.communicate()
        if (out is not None) and isinstance(out, bytes):
            out = out.decode()
        if (err is not None) and isinstance(err, bytes):
            err = err.decode()
        rc = process.returncode
    
        if rc != 0:
>           raise ToolError(cmd_args, rc, out, err)
E           ccmlib.node.ToolError: Subprocess ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']] exited with non-zero status; exit status: 2; 
E           stdout: Consistency level set to ALL.
E           TRACING set to ON
E           ; 
E           stderr: :4:WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received only 2 responses." info={'consistency': 'ALL', 'required_responses': 3, 'received_responses': 2, 'write_type': 'SIMPLE'}

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:2333: ToolError

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

ccmlib.node.ToolError: Subprocess ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']] exited with non-zero status; exit status: 2; 
stdout: Consistency level set to ALL.
TRACING set to ON
; 
stderr: :4:WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received only 2 responses." info={'consistency': 'ALL', 'required_responses': 3, 'received_responses': 2, 'write_type': 'SIMPLE'}-self = 

    @since('2.2')
    def test_tracing_simple(self):
        """
        Test tracing using the default tracing class. See trace().
    
        @jira_ticket CASSANDRA-10392
        @jira_ticket CASSANDRA-11598
        # Restricted to 2.2+ due to flakiness on 2.1.  See CASSANDRA-11598 and CASSANDRA-12407 for details.
        """
        session = self.prepare()
>       self.trace(session)

cql_tracing_test.py:121: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
cql_tracing_test.py:82: in trace
    out, err, _ = node1.run_cqlsh(
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:1122: in run_cqlsh
    return handle_external_tool_process(p, ['cqlsh', cmds, cqlsh_options])
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

process = 
cmd_args = ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']]

    def handle_external_tool_process(process, cmd_args):
        out, err = process.communicate()
        if (out is not None) and isinstance(out, bytes):
            out = out.decode()
        if (err is not None) and isinstance(err, bytes):
            err = err.decode()
        rc = process.returncode
    
        if rc != 0:
>           raise ToolError(cmd_args, rc, out, err)
E           ccmlib.node.ToolError: Subprocess ['cqlsh', "CONSISTENCY ALL; TRACING ON; INSERT INTO ks.users (userid, firstname, lastname, age) VALUES (550e8400-e29b-41d4-a716-446655440000, 'Frodo', 'Baggins', 32)", ['--request-timeout', '30']] exited with non-zero status; exit status: 2; 
E           stdout: Consistency level set to ALL.
E           TRACING set to ON
E           ; 
E           stderr: :4:WriteTimeout: Error from server: code=1100 [Coordinator node timed out waiting for replica nodes' responses] message="Operation timed out - received only 2 responses." info={'consistency': 'ALL', 'required_responses': 3, 'received_responses': 2, 'write_type': 'SIMPLE'}

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:2333: ToolError

cassandra.OperationTimedOut: errors={'Connection defunct by heartbeat': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    @since('3.0')
    def test_tracing_does_not_interfere_with_digest_calculation(self):
        """
        Test that enabling tracing doesn't interfere with digest responses when using RandomPartitioner.
        The use of a threadlocal MessageDigest for generating both DigestResponse messages and for
        calculating tokens meant that the DigestResponse was always incorrect when both RP and tracing
        were enabled, leading to unnecessary data reads.
    
        @jira_ticket CASSANDRA-13964
        """
        cluster = self.cluster
        cluster.populate(3)
        cluster.set_configuration_options(values={'write_request_timeout_in_ms': 30000,
                                                  'read_request_timeout_in_ms': 30000})
        cluster.set_partitioner("org.apache.cassandra.dht.RandomPartitioner")
        cluster.start(jvm_args=['-Dcassandra.wait_for_tracing_events_timeout_secs=15'])
    
        node1 = cluster.nodelist()[0]
        session = self.patient_cql_connection(node1)
        create_ks(session, 'ks', 3)
    
        session.execute("""
            CREATE TABLE ks.users (
                userid uuid PRIMARY KEY,
                firstname text,
                lastname text,
                age int
            );
        """)
    
        insert = session.prepare(
              "INSERT INTO ks.users (userid, firstname, lastname, age) "
              "VALUES (?, 'Frodo', 'Baggins', 32)")
        insert.consistency_level = ConsistencyLevel.ALL
    
        select = session.prepare(
              "SELECT firstname, lastname "
              "FROM ks.users WHERE userid = ?")
        select.consistency_level = ConsistencyLevel.ALL
    
        for _ in range(10):
            id = uuid4()
            session.execute(insert.bind((id,)), timeout=30)
>           res = session.execute(select.bind((id,)), timeout=30, trace=True)

read_repair_test.py:427: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'Connection defunct by heartbeat': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    def test_rename_column_atomicity(self):
        """
        Test that column renaming is atomically done between a table and its materialized views
        @jira_ticket CASSANDRA-12952
        """
        session = self.prepare(nodes=1, user_table=True, install_byteman=True)
        node = self.cluster.nodelist()[0]
    
        self._insert_data(session)
    
        assert_one(
            session,
            "SELECT * FROM users_by_state WHERE state = 'TX' AND username = 'user1'",
            ['TX', 'user1', 1968, 'f', 'ch@ngem3a', None]
        )
    
        # Rename a column with an injected byteman rule to kill the node after the first schema update
        self.fixture_dtest_setup.allow_log_errors = True
    
        script_version = '5_1' if self.cluster.version() >= LooseVersion('5.1') else '4x' if self.cluster.version() >= '4' else '3x'
        node.byteman_submit([mk_bman_path('merge_schema_failure_{}.btm'.format(script_version))])
        with pytest.raises(NoHostAvailable):
>           session.execute("ALTER TABLE users RENAME username TO user")

materialized_views_test.py:994: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.1:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

cassandra.OperationTimedOut: errors={'Connection defunct by heartbeat': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042-self = 

    @since('3.0')
    def test_tracing_does_not_interfere_with_digest_calculation(self):
        """
        Test that enabling tracing doesn't interfere with digest responses when using RandomPartitioner.
        The use of a threadlocal MessageDigest for generating both DigestResponse messages and for
        calculating tokens meant that the DigestResponse was always incorrect when both RP and tracing
        were enabled, leading to unnecessary data reads.
    
        @jira_ticket CASSANDRA-13964
        """
        cluster = self.cluster
        cluster.populate(3)
        cluster.set_configuration_options(values={'write_request_timeout_in_ms': 30000,
                                                  'read_request_timeout_in_ms': 30000})
        cluster.set_partitioner("org.apache.cassandra.dht.RandomPartitioner")
        cluster.start(jvm_args=['-Dcassandra.wait_for_tracing_events_timeout_secs=15'])
    
        node1 = cluster.nodelist()[0]
        session = self.patient_cql_connection(node1)
        create_ks(session, 'ks', 3)
    
        session.execute("""
            CREATE TABLE ks.users (
                userid uuid PRIMARY KEY,
                firstname text,
                lastname text,
                age int
            );
        """)
    
        insert = session.prepare(
              "INSERT INTO ks.users (userid, firstname, lastname, age) "
              "VALUES (?, 'Frodo', 'Baggins', 32)")
        insert.consistency_level = ConsistencyLevel.ALL
    
        select = session.prepare(
              "SELECT firstname, lastname "
              "FROM ks.users WHERE userid = ?")
        select.consistency_level = ConsistencyLevel.ALL
    
        for _ in range(10):
            id = uuid4()
            session.execute(insert.bind((id,)), timeout=30)
>           res = session.execute(select.bind((id,)), timeout=30, trace=True)

read_repair_test.py:427: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'Connection defunct by heartbeat': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.1:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.-junit.framework.AssertionFailedError: Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1365)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1365)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor49.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1365)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.apache.cassandra.anttasks.TestHelper.execute(TestHelper.java:53)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1365)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)