CI Summary

Failed: Timeout >900.0s-self = 

    def test_multi_partition_consistent_reads_after_write(self):
        """
        Tests consistency of multiple writes to a multiple partitions
    
        @jira_ticket CASSANDRA-10981
        """
>       self._consistent_reads_after_write_test(5)

materialized_views_test.py:2846: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 
num_partitions = 5

    def _consistent_reads_after_write_test(self, num_partitions):
    
        session = self.prepare()
        node1, node2, node3 = self.cluster.nodelist()
    
        # Test config
        lower = 0
        upper = 100000
        processes = 4
        queues = [None] * processes
        eachProcess = (upper - lower) // processes
    
        logger.debug("Creating schema")
        session.execute(
            ("CREATE KEYSPACE IF NOT EXISTS mvtest WITH replication = "
             "{'class': 'SimpleStrategy', 'replication_factor': '3'}")
        )
        session.execute(
            "CREATE TABLE mvtest.test1 (a int, b int, c int, d int, PRIMARY KEY (a,b))"
        )
        session.cluster.control_connection.wait_for_schema_agreement()
    
        insert1 = session.prepare("INSERT INTO mvtest.test1 (a,b,c,d) VALUES (?,?,?,?)")
        insert1.consistency_level = writeConsistency
    
        logger.debug("Writing data to base table")
        for i in range(upper // 10):
            self._do_row(insert1, i, num_partitions)
    
        logger.debug("Creating materialized view")
        session.execute(
            ('CREATE MATERIALIZED VIEW mvtest.mv1 AS '
             'SELECT a,b,c,d FROM mvtest.test1 WHERE a IS NOT NULL AND b IS NOT NULL AND '
             'c IS NOT NULL PRIMARY KEY (c,a,b)')
        )
        session.cluster.control_connection.wait_for_schema_agreement()
    
        logger.debug("Writing more data to base table")
        for i in range(upper // 10, upper):
            self._do_row(insert1, i, num_partitions)
    
        # Wait that all requests are done
        while self.num_request_done < upper:
>           time.sleep(1)
E           Failed: Timeout >900.0s

materialized_views_test.py:2891: Failed

cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042-self = 

    @since('2.1.1', max_version='5.0.x')
    def test_simultaneous_bootstrap(self):
        """
        Attempt to bootstrap two nodes at once, to assert the second bootstrapped node fails, and does not interfere.
    
        Start a one node cluster and run a stress write workload.
        Start up a second node, and wait for the first node to detect it has joined the cluster.
        While the second node is bootstrapping, start a third node. This should fail.
    
        @jira_ticket CASSANDRA-7069
        @jira_ticket CASSANDRA-9484
        """
    
        bootstrap_error = "Other bootstrapping/leaving/moving nodes detected," \
                          " cannot bootstrap while cassandra.consistent.rangemovement is true"
    
        cluster = self.cluster
        cluster.set_environment_variable('CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
        cluster.populate(1)
        cluster.start()
    
        node1, = cluster.nodelist()
    
        node1.stress(['write', 'n=500K', 'no-warmup', '-schema', 'replication(factor=1)',
                      '-rate', 'threads=10'])
    
        node2 = new_node(cluster)
        node2.start()
    
        for _ in range(30):  # wait until node2 shows up
            ntout = node1.nodetool('status').stdout
            if re.search(r'UJ\s+' + node2.ip_addr, ntout):
                break
            time.sleep(0.1)
    
        node3 = new_node(cluster, remote_debug_port='2003')
        try:
            node3.start(wait_other_notice=False, verbose=False)
        except NodeError:
            pass  # node doesn't start as expected
    
        time.sleep(.5)
        node2.watch_log_for("Starting listening for CQL clients")
    
        node3.watch_log_for(bootstrap_error)
    
        session = self.patient_exclusive_cql_connection(node2)
    
        # Repeat the select count(*) query, to help catch
        # bugs like 9484, where count(*) fails at higher
        # data loads.
        for _ in range(5):
>           assert_one(session, "SELECT count(*) from keyspace1.standard1", [500000], cl=ConsistencyLevel.ONE)

bootstrap_test.py:932: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tools/assertions.py:130: in assert_one
    res = session.execute(simple_query)
../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:2618: in execute
    return self.execute_async(query, parameters, trace, custom_payload, timeout, execution_profile, paging_state, host, execute_as).result()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = 

    def result(self):
        """
        Return the final result or raise an Exception if errors were
        encountered.  If the final result or error has not been set
        yet, this method will block until it is set, or the timeout
        set for the request expires.
    
        Timeout is specified in the Session request execution functions.
        If the timeout is exceeded, an :exc:`cassandra.OperationTimedOut` will be raised.
        This is a client-side timeout. For more information
        about server-side coordinator timeouts, see :class:`.policies.RetryPolicy`.
    
        Example usage::
    
            >>> future = session.execute_async("SELECT * FROM mycf")
            >>> # do other stuff...
    
            >>> try:
            ...     rows = future.result()
            ...     for row in rows:
            ...         ... # process results
            ... except Exception:
            ...     log.exception("Operation failed:")
    
        """
        self._event.wait()
        if self._final_result is not _NOT_SET:
            return ResultSet(self, self._final_result)
        else:
>           raise self._final_exception
E           cassandra.OperationTimedOut: errors={'127.0.0.2:9042': 'Client request timeout. See Session.execute[_async](timeout)'}, last_host=127.0.0.2:9042

../cassandra/build/venv/src/cassandra-driver/cassandra/cluster.py:4894: OperationTimedOut

Failed on seed 0xe8dcd5b8d4090381-org.apache.cassandra.simulator.SimulationException: Failed on seed 0xe8dcd5b8d4090381
	Suppressed: java.lang.RuntimeException: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.Throwables.maybeFail(Throwables.java:79)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:548)
		at org.apache.cassandra.distributed.impl.AbstractCluster.close(AbstractCluster.java:1089)
		at org.apache.cassandra.simulator.ClusterSimulation.close(ClusterSimulation.java:831)
		at org.apache.cassandra.simulator.SimulationRunner$Run.run(SimulationRunner.java:363)
		at org.apache.cassandra.simulator.SimulationRunner$BasicCommand.run(SimulationRunner.java:348)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner$Run.run(PaxosSimulationRunner.java:34)
		at org.apache.cassandra.simulator.paxos.PaxosSimulationRunner.main(PaxosSimulationRunner.java:148)
		at org.apache.cassandra.simulator.test.ShortPaxosSimulationTest.simulationTest(ShortPaxosSimulationTest.java:33)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
		at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
		at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	Caused by: java.util.concurrent.TimeoutException
		at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:253)
		at org.apache.cassandra.utils.FBUtilities.waitOnFutures(FBUtilities.java:540)
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.ExecutionException: java.util.concurrent.ExecutionException: FSWriteError in /cassandra/node3/commitlog/CommitLog-7-1577836802875.log
			at org.apache.cassandra.utils.concurrent.AbstractFuture.getWhenDone(AbstractFuture.java:239)
			at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:254)
		Caused by: java.util.concurrent.ExecutionException: FSWriteError in /cassandra/node3/commitlog/CommitLog-7-1577836802875.log
			at org.apache.cassandra.utils.concurrent.AbstractFuture.getWhenDone(AbstractFuture.java:239)
			at org.apache.cassandra.utils.concurrent.AbstractFuture.get(AbstractFuture.java:246)
			at org.apache.cassandra.distributed.impl.Instance.lambda$shutdown$48(Instance.java:957)
			at org.apache.cassandra.concurrent.SyncFutureTask.run(SyncFutureTask.java:68)
			at org.apache.cassandra.simulator.systems.SimulatedExecution$NoIntercept$1Run.run(SimulatedExecution.java:82)
			at org.apache.cassandra.simulator.systems.InterceptingExecutor$InterceptingPooledExecutor$WaitingThread.lambda$new$1(InterceptingExecutor.java:317)
			at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
			at java.base/java.lang.Thread.run(Thread.java:829)
		Caused by: FSWriteError in /cassandra/node3/commitlog/CommitLog-7-1577836802875.log
			at org.apache.cassandra.db.commitlog.MemoryMappedSegment.flush(MemoryMappedSegment.java:97)
			at org.apache.cassandra.db.commitlog.CommitLogSegment.sync$unsync(CommitLogSegment.java:359)
			at org.apache.cassandra.db.commitlog.CommitLogSegment.sync(CommitLogSegment.java)
			at org.apache.cassandra.db.commitlog.CommitLogSegment.close$unsync(CommitLogSegment.java:512)
			at org.apache.cassandra.db.commitlog.CommitLogSegment.close(CommitLogSegment.java)
			at org.apache.cassandra.db.commitlog.AbstractCommitLogSegmentManager.awaitTermination(AbstractCommitLogSegmentManager.java:591)
			at org.apache.cassandra.db.commitlog.CommitLog.shutdownBlocking$unsync(CommitLog.java:504)
			at org.apache.cassandra.db.commitlog.CommitLog.shutdownBlocking(CommitLog.java)
			at org.apache.cassandra.distributed.impl.Instance.lambda$parallelRun$51(Instance.java:1168)
		Caused by: java.io.UncheckedIOException: java.nio.channels.ClosedChannelException
			at org.apache.cassandra.io.filesystem.ListenableFileSystem$ListenableFileChannel.lambda$map$19(ListenableFileSystem.java:809)
			at org.apache.cassandra.utils.SyncUtil.force(SyncUtil.java:66)
			at org.apache.cassandra.db.commitlog.MemoryMappedSegment.flush(MemoryMappedSegment.java:93)
		Caused by: java.nio.channels.ClosedChannelException
			at com.google.common.jimfs.JimfsFileChannel.checkOpen(JimfsFileChannel.java:112)
			at com.google.common.jimfs.JimfsFileChannel.write(JimfsFileChannel.java:539)
			at org.apache.cassandra.io.filesystem.ForwardingFileChannel.write(ForwardingFileChannel.java:118)
			at org.apache.cassandra.io.filesystem.ListenableFileSystem$ListenableFileChannel.write(ListenableFileSystem.java:709)
			at org.apache.cassandra.io.filesystem.ListenableFileSystem$ListenableFileChannel.lambda$map$19(ListenableFileSystem.java:801)
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
		Suppressed: java.util.concurrent.TimeoutException
Caused by: java.lang.OutOfMemoryError: Java heap space
	at org.apache.cassandra.utils.JVMStabilityInspector.forceHeapSpaceOomMaybe(JVMStabilityInspector.java:193)
	at org.apache.cassandra.utils.JVMStabilityInspector.inspectThrowable(JVMStabilityInspector.java:127)
	at org.apache.cassandra.utils.JVMStabilityInspector.inspectThrowable(JVMStabilityInspector.java:170)
	at org.apache.cassandra.utils.JVMStabilityInspector.inspectThrowable(JVMStabilityInspector.java:89)
	at org.apache.cassandra.streaming.StreamDeserializingTask.run(StreamDeserializingTask.java:80)
	at org.apache.cassandra.simulator.systems.InterceptedExecution$InterceptedThreadStart.run(InterceptedExecution.java:216)
	at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
	at java.base/java.lang.Thread.run(Thread.java:829)