CI Summary

AssertionError: Commitlog was written with durable writes disabled
assert 90112 == 86016
  +90112
  -86016-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 90112 == 86016
E         +90112
E         -86016

configuration_test.py:104: AssertionError

AssertionError: Commitlog was written with durable writes disabled
assert 90112 == 86016
  +90112
  -86016-self = 

    @pytest.mark.timeout(60*30)
    def test_change_durable_writes(self):
        """
        @jira_ticket CASSANDRA-9560
    
        Test that changes to the DURABLE_WRITES option on keyspaces is
        respected in subsequent writes.
    
        This test starts by writing a dataset to a cluster and asserting that
        the commitlogs have been written to. The subsequent test depends on
        the assumption that this dataset triggers an fsync.
    
        After checking this assumption, the test destroys the cluster and
        creates a fresh one. Then it tests that DURABLE_WRITES is respected by:
    
        - creating a keyspace with DURABLE_WRITES set to false,
        - using ALTER KEYSPACE to set its DURABLE_WRITES option to true,
        - writing a dataset to this keyspace that is known to trigger a commitlog fsync,
        - asserting that the commitlog has grown in size since the data was written.
        """
        cluster = self.cluster
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
    
        cluster.populate(1).start()
        durable_node = cluster.nodelist()[0]
    
        durable_init_size = commitlog_size(durable_node)
        durable_session = self.patient_exclusive_cql_connection(durable_node)
    
        # test assumption that write_to_trigger_fsync actually triggers a commitlog fsync
        durable_session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                                "AND DURABLE_WRITES = true")
        durable_session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
        write_to_trigger_fsync(durable_session, 'ks', 'tab')
        logger.debug('commitlog size diff = ' + str(commitlog_size(durable_node) - durable_init_size))
    
        assert commitlog_size(durable_node) > durable_init_size, \
            "This test will not work in this environment; write_to_trigger_fsync does not trigger fsync."
    
        durable_session.shutdown()
        cluster.stop()
        cluster.clear()
    
        cluster.set_batch_commitlog(enabled=True, use_batch_window = cluster.version() < '5.0')
        cluster.set_configuration_options(values={'commitlog_segment_size_in_mb': 1})
        cluster.start()
        node = cluster.nodelist()[0]
        session = self.patient_exclusive_cql_connection(node)
    
        # set up a keyspace without durable writes, then alter it to use them
        session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': 1} "
                        "AND DURABLE_WRITES = false")
        session.execute('CREATE TABLE ks.tab (key int PRIMARY KEY, a int, b int, c int)')
        init_size = commitlog_size(node)
        write_to_trigger_fsync(session, 'ks', 'tab')
>       assert commitlog_size(node) == init_size, "Commitlog was written with durable writes disabled"
E       AssertionError: Commitlog was written with durable writes disabled
E       assert 90112 == 86016
E         +90112
E         -86016

configuration_test.py:104: AssertionError

RuntimeError: A subprocess has terminated early. Subprocess statuses: data_writer (is_alive: False, exitCode: 1), data_checker (is_alive: True, exitCode: None), attempting to terminate remaining subprocesses now.-self = 

    @pytest.mark.timeout(3000)
    def test_rolling_upgrade(self):
        """
        Test rolling upgrade of the cluster, so we have mixed versions part way through.
        """
>       self.upgrade_scenario(rolling=True)

upgrade_tests/upgrade_through_versions_test.py:395: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
upgrade_tests/upgrade_through_versions_test.py:471: in upgrade_scenario
    self._check_on_subprocs(self.fixture_dtest_setup.subprocs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 
subprocs = [, ]

    def _check_on_subprocs(self, subprocs):
        """
        Check on given subprocesses.
    
        If any are not alive, we'll go ahead and terminate any remaining alive subprocesses since this test is going to fail.
        """
        subproc_statuses = [s.is_alive() for s in subprocs]
        if not all(subproc_statuses):
            message = "A subprocess has terminated early. Subprocess statuses: "
            for s in subprocs:
                message += "{name} (is_alive: {aliveness}, exitCode: {exitCode}), ".format(name=s.name, aliveness=s.is_alive(), exitCode=s.exitcode)
            message += "attempting to terminate remaining subprocesses now."
            self._terminate_subprocs()
>           raise RuntimeError(message)
E           RuntimeError: A subprocess has terminated early. Subprocess statuses: data_writer (is_alive: False, exitCode: 1), data_checker (is_alive: True, exitCode: None), attempting to terminate remaining subprocesses now.

upgrade_tests/upgrade_through_versions_test.py:532: RuntimeError

junit.framework.AssertionFailedError
	at org.apache.cassandra.db.DiskBoundaryManagerTest.updateTokensTest(DiskBoundaryManagerTest.java:129)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

junit.framework.AssertionFailedError
	at org.apache.cassandra.db.commitlog.CommitlogShutdownTest.testShutdownWithPendingTasks(CommitlogShutdownTest.java:97)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$10.evaluate(BMUnitRunner.java:393)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$6.evaluate(BMUnitRunner.java:263)
	at org.jboss.byteman.contrib.bmunit.BMUnitRunner$1.evaluate(BMUnitRunner.java:97)

java.lang.OutOfMemoryError: Java heap space-java.lang.RuntimeException: java.lang.OutOfMemoryError: Java heap space
	at org.apache.cassandra.distributed.impl.Instance.lambda$startup$7(Instance.java:640)
	at org.apache.cassandra.concurrent.FutureTask$1.call(FutureTask.java:96)
	at org.apache.cassandra.concurrent.FutureTask.call(FutureTask.java:61)
	at org.apache.cassandra.concurrent.FutureTask.run(FutureTask.java:71)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: java.lang.OutOfMemoryError: Java heap space

All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Channel has been closed))-com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /127.0.0.1:9042 (com.datastax.driver.core.exceptions.TransportException: [/127.0.0.1:9042] Channel has been closed))
	at com.datastax.driver.core.ControlConnection.reconnectInternal(ControlConnection.java:270)
	at com.datastax.driver.core.ControlConnection.connect(ControlConnection.java:109)
	at com.datastax.driver.core.Cluster$Manager.negotiateProtocolVersionAndConnect(Cluster.java:1813)
	at com.datastax.driver.core.Cluster$Manager.init(Cluster.java:1726)
	at com.datastax.driver.core.Cluster.init(Cluster.java:214)
	at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:387)
	at com.datastax.driver.core.Cluster.connectAsync(Cluster.java:366)
	at com.datastax.driver.core.Cluster.connect(Cluster.java:311)
	at org.apache.cassandra.distributed.test.NativeTransportEncryptionOptionsTest.testOptionalMtlsModeDoNotAllowNonSSLConnections(NativeTransportEncryptionOptionsTest.java:300)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

SHOULD NOT BE ON NODE: 11 -- [(16,30)]: [00, 02, 04, 06, 08, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 42, 44, 46, 48]-junit.framework.AssertionFailedError: SHOULD NOT BE ON NODE: 11 -- [(16,30)]: [00, 02, 04, 06, 08, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 42, 44, 46, 48]
	at org.apache.cassandra.distributed.test.TransientRangeMovementTest.assertAllContained(TransientRangeMovementTest.java:231)
	at org.apache.cassandra.distributed.test.TransientRangeMovement2Test.testMoveForward(TransientRangeMovement2Test.java:143)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.-junit.framework.AssertionFailedError: Forked Java VM exited abnormally. Please note the time in the report does not reflect the time until the VM exit.
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.base/java.util.Vector.forEach(Vector.java:1394)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)

expected:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]> but was:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]>-junit.framework.AssertionFailedError: expected:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.5:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]> but was:<[/127.0.100.1:7012, /127.0.100.2:7012, /127.0.100.3:7012, /127.0.100.4:7012, /127.0.100.6:7012, /127.0.100.7:7012, /127.0.100.8:7012, /127.0.100.9:7012, /127.0.100.10:7012]>
	at org.apache.cassandra.tcm.DiscoverySimulationTest.discoveryTest(DiscoverySimulationTest.java:105)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
	at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)