CI Summary

failed on teardown with "Unexpected error found in node logs (see stdout for full details). Errors: [[node2] 'ERROR [Messaging-EventLoop-3-3] 2024-04-29 15:48:34,121 NoSpamLogger.java:110 - /127.0.0.2:7000->/127.0.0.1:7000-URGENT_MESSAGES-[no-channel] failed to connect\njava.nio.channels.ClosedChannelException: null\n\tat org.apache.cassandra.net.OutboundConnectionInitiator$Handler.channelInactive(OutboundConnectionInitiator.java:322)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376)\n\tat io.netty.handler.ssl.SslHandler.channelInactive(SslHandler.java:1087)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813)\n\tat io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)\n\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)\n\tat io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:413)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:829)']"-Unexpected error found in node logs (see stdout for full details). Errors: [[node2] 'ERROR [Messaging-EventLoop-3-3] 2024-04-29 15:48:34,121 NoSpamLogger.java:110 - /127.0.0.2:7000->/127.0.0.1:7000-URGENT_MESSAGES-[no-channel] failed to connect\njava.nio.channels.ClosedChannelException: null\n\tat org.apache.cassandra.net.OutboundConnectionInitiator$Handler.channelInactive(OutboundConnectionInitiator.java:322)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:411)\n\tat io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:376)\n\tat io.netty.handler.ssl.SslHandler.channelInactive(SslHandler.java:1087)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274)\n\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301)\n\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281)\n\tat io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)\n\tat io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813)\n\tat io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:174)\n\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:167)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470)\n\tat io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:413)\n\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997)\n\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n\tat java.base/java.lang.Thread.run(Thread.java:829)']

ccmlib.node.TimeoutError: 28 Apr 2024 22:20:23 [node1] after 90.09/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
 Head: INFO  [main] 2024-04-28 22:18:54,662 YamlConfigura
 Tail: ...able: /home/cassandra/cassandra/build/run-python-dtest.GH44V5/dtest-1kuh4d3n/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-22-big-self = 

    @pytest.mark.timeout(3000)
    def test_parallel_upgrade_with_internode_ssl(self):
        """
        Test upgrading cluster all at once (requires cluster downtime), with internode ssl.
        """
>       self.upgrade_scenario(internode_ssl=True)

upgrade_tests/upgrade_through_versions_test.py:402: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
upgrade_tests/upgrade_through_versions_test.py:493: in upgrade_scenario
    self.upgrade_to_version(version_meta, internode_ssl=internode_ssl)
upgrade_tests/upgrade_through_versions_test.py:581: in upgrade_to_version
    node.start(wait_other_notice=400, wait_for_binary_proto=True,
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:922: in start
    self.wait_for_binary_interface(from_mark=self.mark)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:710: in wait_for_binary_interface
    self.watch_log_for("Starting listening for CQL clients", **kwargs)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:609: in watch_log_for
    TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

start = 1714342733.1594653, timeout = 90
msg = "Missing: ['Starting listening for CQL clients'] not found in system.log:\n Head: INFO  [main] 2024-04-28 22:18:54,662...uild/run-python-dtest.GH44V5/dtest-1kuh4d3n/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-22-big\n"
node = 'node1'

    @staticmethod
    def raise_if_passed(start, timeout, msg, node=None):
        if start + timeout < time.time():
>           raise TimeoutError.create(start, timeout, msg, node)
E           ccmlib.node.TimeoutError: 28 Apr 2024 22:20:23 [node1] after 90.09/90 seconds Missing: ['Starting listening for CQL clients'] not found in system.log:
E            Head: INFO  [main] 2024-04-28 22:18:54,662 YamlConfigura
E            Tail: ...able: /home/cassandra/cassandra/build/run-python-dtest.GH44V5/dtest-1kuh4d3n/test/node1/data0/system/local-7ad54392bcdd35a684174e047860b377/oa-22-big

../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:56: TimeoutError