8_cythonno_x86_64_58_64 | test_dc_parallel_repair | Failure | Failed: Timeout >900.0s
self = <repair_tests.repair_test.TestRepair object at 0x7f0de72e9760>
def test_dc_parallel_repair(self):
"""
* Set up a multi DC cluster
* Perform a -dc repair on two dc's, with -dcpar
* Assert only nodes on those dcs were repaired
"""
cluster = self._setup_multi_dc()
node1 = cluster.nodes["node1"]
node2 = cluster.nodes["node2"]
node3 = cluster.nodes["node3"]
logger.debug("starting repair...")
opts = ["-dc", "dc1", "-dc", "dc2", "-dcpar"]
opts += _repair_options(self.cluster.version(), ks="ks", sequential=False)
node1.repair(opts)
# Verify that only nodes in dc1 and dc2 are involved in repair
out_of_sync_logs = node1.grep_log(r"{} and {} have ([0-9]+) range\(s\) out of sync".format(cluster.address_regex(), cluster.address_regex()))
assert len(out_of_sync_logs) == 2, "Lines matching: " + str([elt[0] for elt in out_of_sync_logs])
valid_out_of_sync_pairs = [{node1.address(), node2.address()},
{node2.address(), node3.address()}]
for line, m in out_of_sync_logs:
num_out_of_sync_ranges, out_of_sync_nodes = m.group(3), {m.group(1), m.group(2)}
assert int(num_out_of_sync_ranges) == 1, "Expecting 1 range out of sync for {}, but saw {}".format(out_of_sync_nodes, num_out_of_sync_ranges)
assert out_of_sync_nodes, valid_out_of_sync_pairs in str(out_of_sync_nodes)
# Check node2 now has the key
> self.check_rows_on_node(node2, 2001, found=[1000], restart=False)
repair_tests/repair_test.py:704:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
repair_tests/repair_test.py:74: in check_rows_on_node
node.stop(wait_other_notice=True)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:987: in stop
node.watch_log_for_death(self, from_mark=mark)
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:677: in watch_log_for_death
self.watch_log_for(tofind, from_mark=from_mark, timeout=timeout, filename=filename)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ccmlib.node.Node object at 0x7f0de5843d00>
exprs = ['127.0.0.4:7000 is now [dead|DOWN]'], from_mark = 549702, timeout = 600
process = None, verbose = False, filename = 'system.log'
error_on_pid_terminated = False
def watch_log_for(self, exprs, from_mark=None, timeout=600,
process=None, verbose=False, filename='system.log',
error_on_pid_terminated=False):
"""
Watch the log until one or more (regular) expressions are found or timeouts (a
TimeoutError is then raised). On successful completion, a list of pair (line matched,
match object) is returned.
Will raise NodeError if error_on_pit_terminated is True and C* pid is not running.
"""
start = time.time()
tofind = [exprs] if isinstance(exprs, string_types) else exprs
tofind = [re.compile(e) for e in tofind]
matchings = []
reads = ""
if len(tofind) == 0:
return None
log_file = os.path.join(self.log_directory(), filename)
output_read = False
while not os.path.exists(log_file):
time.sleep(.5)
TimeoutError.raise_if_passed(start=start, timeout=timeout, node=self.name,
msg="Timed out waiting for {} to be created.".format(log_file))
if process and not output_read:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
output_read = True
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
with open(log_file) as f:
if from_mark:
f.seek(from_mark)
while True:
# First, if we have a process to check, then check it.
# Skip on Windows - stdout/stderr is cassandra.bat
if not common.is_win() and not output_read:
if process:
process.poll()
if process.returncode is not None:
self.print_process_output(self.name, process, verbose)
output_read = True
if process.returncode != 0:
raise RuntimeError() # Shouldn't reuse RuntimeError but I'm lazy
line = f.readline()
if line:
reads = reads + line
for e in tofind:
m = e.search(line)
if m:
matchings.append((line, m))
tofind.remove(e)
if len(tofind) == 0:
return matchings[0] if isinstance(exprs, string_types) else matchings
else:
# wait for the situation to clarify, either stop or just a pause in log production
> time.sleep(1)
E Failed: Timeout >900.0s
../cassandra/build/venv/lib/python3.8/site-packages/ccmlib/node.py:605: Failed | 904.377 |