Class | Name | Status | Type | Time(s) |
8_cythonno_x86_64_1_8 | test_network_topology_strategy | Success | | 290.002 |
8_cythonno_x86_64_1_8 | test_disk_balance_bootstrap | Success | | 128.776 |
8_cythonno_x86_64_1_8 | test_throttled_partition_update | Success | | 564.371 |
8_cythonno_x86_64_1_8 | test_resume_failed_replace | Success | | 181.933 |
8_cythonno_x86_64_1_8 | test_initial_empty_repair_tables | Skipped | hangs CI
/home/cassandra/cassandra-dtest/repair_tests/repair_test.py:1416: hangs CI | 0.001 |
8_cythonno_x86_64_2_8 | test_network_topology_strategy_each_quorum | Success | | 162.905 |
8_cythonno_x86_64_2_8 | test_add_dc_after_mv_simple_replication | Success | | 98.469 |
8_cythonno_x86_64_2_8 | test_really_complex_repair | Success | | 265.850 |
8_cythonno_x86_64_2_8 | test_restart_failed_replace_with_reset_resume_state | Success | | 193.274 |
8_cythonno_x86_64_2_8 | test_repair_parent_table | Success | | 72.072 |
8_cythonno_x86_64_3_8 | test_simple_strategy_users | Success | | 149.232 |
8_cythonno_x86_64_3_8 | test_add_dc_after_mv_network_replication | Success | | 100.055 |
8_cythonno_x86_64_3_8 | test_replace_stopped_node | Success | | 181.615 |
8_cythonno_x86_64_3_8 | test_restart_failed_replace | Success | | 251.583 |
8_cythonno_x86_64_3_8 | test_repair_table | Success | | 73.763 |
8_cythonno_x86_64_4_8 | test_simple_strategy_each_quorum_users | Success | | 75.073 |
8_cythonno_x86_64_4_8 | test_add_node_after_mv | Success | | 74.757 |
8_cythonno_x86_64_4_8 | test_replace_shutdown_node | Success | | 184.897 |
8_cythonno_x86_64_4_8 | test_rf_collapse_gossiping_property_file_snitch_multi_dc | Skipped | 5.1 > 5.0.x
/home/cassandra/cassandra-dtest/conftest.py:468: 5.1 > 5.0.x | 0.224 |
8_cythonno_x86_64_5_8 | test_network_topology_strategy_users | Success | | 161.390 |
8_cythonno_x86_64_5_8 | test_add_node_after_wide_mv_with_range_deletions | Success | | 98.110 |
8_cythonno_x86_64_5_8 | test_replace_stopped_node_same_address | Success | | 193.783 |
8_cythonno_x86_64_5_8 | test_rf_expand_gossiping_property_file_snitch_multi_dc | Skipped | 5.1 > 5.0.x
/home/cassandra/cassandra-dtest/conftest.py:468: 5.1 > 5.0.x | 0.339 |
8_cythonno_x86_64_6_8 | test_network_topology_strategy_each_quorum_users | Success | | 76.931 |
8_cythonno_x86_64_6_8 | test_add_node_after_very_wide_mv | Success | | 265.716 |
8_cythonno_x86_64_6_8 | test_replace_first_boot | Success | | 195.571 |
8_cythonno_x86_64_6_8 | test_multidatacenter_local_quorum | Failure | AssertionError: too many reads hit the degraded node: 4
assert 4 <= 1
self = <snitch_test.TestDynamicEndpointSnitch object at 0x7f3703a950d0>
@pytest.mark.resource_intensive
@since('3.10')
def test_multidatacenter_local_quorum(self):
'''
@jira_ticket CASSANDRA-13074
If we do only local datacenters reads in a multidatacenter DES setup,
DES should take effect and route around a degraded node
'''
def no_cross_dc(scores, cross_dc_nodes):
return all('/' + k.address() not in scores for k in cross_dc_nodes)
def snitchable(scores_before, scores_after, needed_nodes):
return all('/' + k.address() in scores_before and '/' + k.address()
in scores_after for k in needed_nodes)
cluster = self.cluster
cluster.populate([3, 3])
coordinator_node, healthy_node, degraded_node, node4, node5, node6 = cluster.nodelist()
# increase DES reset/update interval so we clear any cross-DC startup reads faster
cluster.set_configuration_options(values={'dynamic_snitch_reset_interval_in_ms': 10000,
'dynamic_snitch_update_interval_in_ms': 50,
'phi_convict_threshold': 12})
# Delay reads on the degraded node by 50 milliseconds
degraded_node.start(jvm_args=['-Dcassandra.test.read_iteration_delay_ms=50',
'-Dcassandra.allow_unsafe_join=true'])
cluster.start()
des = make_mbean('db', type='DynamicEndpointSnitch')
read_stage = make_mbean('metrics', type='ThreadPools', path='request',
scope='ReadStage', name='CompletedTasks')
session = self.patient_exclusive_cql_connection(coordinator_node)
session.execute("CREATE KEYSPACE snitchtestks WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 3, 'dc2': 3}")
session.execute("CREATE TABLE snitchtestks.tbl1 (key int PRIMARY KEY) WITH speculative_retry = 'NONE'")
read_stmt = session.prepare("SELECT * FROM snitchtestks.tbl1 where key = ?")
read_stmt.consistency_level = ConsistencyLevel.LOCAL_QUORUM
insert_stmt = session.prepare("INSERT INTO snitchtestks.tbl1 (key) VALUES (?)")
insert_stmt.consistency_level = ConsistencyLevel.ALL
with JolokiaAgent(coordinator_node) as jmx:
with JolokiaAgent(degraded_node) as bad_jmx:
for x in range(0, 150):
session.execute(insert_stmt, [x])
cleared = False
# Wait for a snitch reset in case any earlier
# startup process populated cross-DC read timings
while not cleared:
scores = jmx.read_attribute(des, 'Scores')
cleared = ('/127.0.0.1' in scores and (len(scores) == 1)) or not scores
snitchable_count = 0
for x in range(0, 150):
degraded_reads_before = bad_jmx.read_attribute(read_stage, 'Value')
scores_before = jmx.read_attribute(des, 'Scores')
assert no_cross_dc(scores_before, [node4, node5, node6]), "Cross DC scores were present: " + str(scores_before)
future = session.execute_async(read_stmt, [x])
future.result()
scores_after = jmx.read_attribute(des, 'Scores')
assert no_cross_dc(scores_after, [node4, node5, node6]), "Cross DC scores were present: " + str(scores_after)
if snitchable(scores_before, scores_after,
[coordinator_node, healthy_node, degraded_node]):
snitchable_count = snitchable_count + 1
# If the DES correctly routed the read around the degraded node,
# it shouldn't have another completed read request in metrics,
# unless there was one right after a reset.
degraded_reads = bad_jmx.read_attribute(read_stage, 'Value')
difference = abs(degraded_reads_before - degraded_reads)
> assert difference <= 1, "too many reads hit the degraded node: %s" % difference
E AssertionError: too many reads hit the degraded node: 4
E assert 4 <= 1
snitch_test.py:235: AssertionError | 147.009 |
8_cythonno_x86_64_7_8 | test_network_topology_strategy_counters | Success | | 141.532 |
8_cythonno_x86_64_7_8 | test_add_write_survey_node_after_mv | Success | | 74.558 |
8_cythonno_x86_64_7_8 | test_replace_active_node | Success | | 90.001 |
8_cythonno_x86_64_7_8 | test_stop_decommission_too_few_replicas_multi_dc | Success | | 59.934 |
8_cythonno_x86_64_8_8 | test_network_topology_strategy_each_quorum_counters | Success | | 97.516 |
8_cythonno_x86_64_8_8 | test_complex_repair | Success | | 322.996 |
8_cythonno_x86_64_8_8 | test_replace_nonexistent_node | Success | | 89.782 |
8_cythonno_x86_64_8_8 | test_sstable_marking_not_intersecting_all_ranges | Success | | 94.198 |
8_cythonno_x86_64_1_8 | test_network_topology_strategy | Success | | 342.099 |
8_cythonno_x86_64_1_8 | test_disk_balance_bootstrap | Success | | 192.892 |
8_cythonno_x86_64_1_8 | test_throttled_partition_update | Success | | 574.091 |
8_cythonno_x86_64_1_8 | test_resume_failed_replace | Success | | 175.719 |
8_cythonno_x86_64_1_8 | test_initial_empty_repair_tables | Skipped | hangs CI
/home/cassandra/cassandra-dtest/repair_tests/repair_test.py:1416: hangs CI | 0.001 |
8_cythonno_x86_64_2_8 | test_network_topology_strategy_each_quorum | Success | | 174.539 |
8_cythonno_x86_64_2_8 | test_add_dc_after_mv_simple_replication | Success | | 97.519 |
8_cythonno_x86_64_2_8 | test_really_complex_repair | Success | | 270.776 |
8_cythonno_x86_64_2_8 | test_restart_failed_replace_with_reset_resume_state | Success | | 191.368 |
8_cythonno_x86_64_2_8 | test_repair_parent_table | Success | | 81.740 |
8_cythonno_x86_64_3_8 | test_simple_strategy_users | Success | | 185.007 |
8_cythonno_x86_64_3_8 | test_add_dc_after_mv_network_replication | Success | | 108.460 |
8_cythonno_x86_64_3_8 | test_replace_stopped_node | Success | | 188.281 |
8_cythonno_x86_64_3_8 | test_restart_failed_replace | Success | | 257.074 |
8_cythonno_x86_64_3_8 | test_repair_table | Success | | 80.232 |
8_cythonno_x86_64_4_8 | test_simple_strategy_each_quorum_users | Success | | 64.328 |
8_cythonno_x86_64_4_8 | test_add_node_after_mv | Success | | 73.711 |
8_cythonno_x86_64_4_8 | test_replace_shutdown_node | Success | | 183.545 |
8_cythonno_x86_64_4_8 | test_rf_collapse_gossiping_property_file_snitch_multi_dc | Skipped | 5.1 > 5.0.x
/home/cassandra/cassandra-dtest/conftest.py:468: 5.1 > 5.0.x | 0.339 |
8_cythonno_x86_64_5_8 | test_network_topology_strategy_users | Success | | 168.033 |
8_cythonno_x86_64_5_8 | test_add_node_after_wide_mv_with_range_deletions | Success | | 85.754 |
8_cythonno_x86_64_5_8 | test_replace_stopped_node_same_address | Success | | 175.155 |
8_cythonno_x86_64_5_8 | test_rf_expand_gossiping_property_file_snitch_multi_dc | Skipped | 5.1 > 5.0.x
/home/cassandra/cassandra-dtest/conftest.py:468: 5.1 > 5.0.x | 0.441 |
8_cythonno_x86_64_6_8 | test_network_topology_strategy_each_quorum_users | Success | | 78.283 |
8_cythonno_x86_64_6_8 | test_add_node_after_very_wide_mv | Success | | 238.739 |
8_cythonno_x86_64_6_8 | test_replace_first_boot | Success | | 183.249 |
8_cythonno_x86_64_6_8 | test_multidatacenter_local_quorum | Success | | 135.875 |
8_cythonno_x86_64_7_8 | test_network_topology_strategy_counters | Success | | 200.725 |
8_cythonno_x86_64_7_8 | test_add_write_survey_node_after_mv | Success | | 86.807 |
8_cythonno_x86_64_7_8 | test_replace_active_node | Success | | 96.216 |
8_cythonno_x86_64_7_8 | test_stop_decommission_too_few_replicas_multi_dc | Success | | 64.613 |
8_cythonno_x86_64_8_8 | test_network_topology_strategy_each_quorum_counters | Success | | 86.547 |
8_cythonno_x86_64_8_8 | test_complex_repair | Success | | 292.171 |
8_cythonno_x86_64_8_8 | test_replace_nonexistent_node | Success | | 79.635 |
8_cythonno_x86_64_8_8 | test_sstable_marking_not_intersecting_all_ranges | Success | | 89.147 |