feat: implement backup cancellation support and add job management UI templates
This commit is contained in:
parent
04fb84c44a
commit
f6f6158811
@ -327,7 +327,8 @@ def upload_via_sftp(host, user, password, key_filename, local_path, remote_dir):
|
|||||||
|
|
||||||
def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False,
|
def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False,
|
||||||
sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None,
|
sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None,
|
||||||
log_path=None, progress_cb=None, disk_filter=None, job_id=None):
|
log_path=None, progress_cb=None, disk_filter=None, job_id=None,
|
||||||
|
is_cancelled_cb=None):
|
||||||
"""Run full backup flow.
|
"""Run full backup flow.
|
||||||
disk_filter: if not None, a set/list of VMDK file-ref strings to include.
|
disk_filter: if not None, a set/list of VMDK file-ref strings to include.
|
||||||
The VMX config file is always included regardless.
|
The VMX config file is always included regardless.
|
||||||
@ -338,7 +339,8 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
|
|||||||
with redirect_stdout(logfile), redirect_stderr(logfile):
|
with redirect_stdout(logfile), redirect_stderr(logfile):
|
||||||
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id)
|
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
|
||||||
|
is_cancelled_cb=is_cancelled_cb)
|
||||||
try:
|
try:
|
||||||
return _wrap()
|
return _wrap()
|
||||||
finally:
|
finally:
|
||||||
@ -346,12 +348,14 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
|
|||||||
else:
|
else:
|
||||||
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id)
|
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
|
||||||
|
is_cancelled_cb=is_cancelled_cb)
|
||||||
|
|
||||||
|
|
||||||
def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||||
progress_cb=None, disk_filter=None, job_id=None):
|
progress_cb=None, disk_filter=None, job_id=None,
|
||||||
|
is_cancelled_cb=None):
|
||||||
def _prog(phase, pct, detail=''):
|
def _prog(phase, pct, detail=''):
|
||||||
if progress_cb:
|
if progress_cb:
|
||||||
try:
|
try:
|
||||||
@ -428,6 +432,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
|||||||
downloaded_files = []
|
downloaded_files = []
|
||||||
files_manifest_info = []
|
files_manifest_info = []
|
||||||
for file_idx, ref in enumerate(all_refs):
|
for file_idx, ref in enumerate(all_refs):
|
||||||
|
if is_cancelled_cb and is_cancelled_cb():
|
||||||
|
raise RuntimeError("Backup cancelled by user")
|
||||||
ds_name, ds_path = parse_datastore_path(ref)
|
ds_name, ds_path = parse_datastore_path(ref)
|
||||||
dc = find_datacenter_for_datastore(content, ds_name)
|
dc = find_datacenter_for_datastore(content, ds_name)
|
||||||
if not dc:
|
if not dc:
|
||||||
@ -441,6 +447,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
|||||||
|
|
||||||
def make_dl_cb(fidx, total, base_pct, share, fname):
|
def make_dl_cb(fidx, total, base_pct, share, fname):
|
||||||
def _dl_cb(done, total_b):
|
def _dl_cb(done, total_b):
|
||||||
|
if is_cancelled_cb and is_cancelled_cb():
|
||||||
|
raise RuntimeError("Backup cancelled by user")
|
||||||
if total_b > 0:
|
if total_b > 0:
|
||||||
file_pct = done / total_b
|
file_pct = done / total_b
|
||||||
overall_pct = int(base_pct + file_pct * share)
|
overall_pct = int(base_pct + file_pct * share)
|
||||||
@ -482,6 +490,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
|||||||
})
|
})
|
||||||
|
|
||||||
_prog('compressing', 90, 'Downloads complete. Creating manifest…')
|
_prog('compressing', 90, 'Downloads complete. Creating manifest…')
|
||||||
|
if is_cancelled_cb and is_cancelled_cb():
|
||||||
|
raise RuntimeError("Backup cancelled by user")
|
||||||
|
|
||||||
# Write manifest.json
|
# Write manifest.json
|
||||||
finished_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
|
finished_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||||
@ -501,6 +511,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
|||||||
|
|
||||||
final_files = []
|
final_files = []
|
||||||
for f in downloaded_files:
|
for f in downloaded_files:
|
||||||
|
if is_cancelled_cb and is_cancelled_cb():
|
||||||
|
raise RuntimeError("Backup cancelled by user")
|
||||||
if compress:
|
if compress:
|
||||||
_prog('compressing', 92, f'Compressing {os.path.basename(f)}…')
|
_prog('compressing', 92, f'Compressing {os.path.basename(f)}…')
|
||||||
cf = maybe_compress(f)
|
cf = maybe_compress(f)
|
||||||
@ -515,6 +527,9 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
|||||||
if not sftp_user:
|
if not sftp_user:
|
||||||
raise Exception('SFTP user required')
|
raise Exception('SFTP user required')
|
||||||
|
|
||||||
|
if is_cancelled_cb and is_cancelled_cb():
|
||||||
|
raise RuntimeError("Backup cancelled by user")
|
||||||
|
|
||||||
# Verify checksums before upload
|
# Verify checksums before upload
|
||||||
_prog('uploading', 94, 'Verifying local checksums before SFTP upload…')
|
_prog('uploading', 94, 'Verifying local checksums before SFTP upload…')
|
||||||
print("Running pre-upload checksum verification...")
|
print("Running pre-upload checksum verification...")
|
||||||
|
|||||||
37
gui_app.py
37
gui_app.py
@ -408,6 +408,8 @@ def run_job_thread(jid):
|
|||||||
info['started'] = time.time()
|
info['started'] = time.time()
|
||||||
info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'}
|
info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'}
|
||||||
|
|
||||||
|
is_cancelled = lambda: jobs.get(jid, {}).get('status') == 'cancelling'
|
||||||
|
|
||||||
vm_names = info.get('vm_names')
|
vm_names = info.get('vm_names')
|
||||||
log_path = str(JOBS_DIR / jid / 'backup.log')
|
log_path = str(JOBS_DIR / jid / 'backup.log')
|
||||||
|
|
||||||
@ -421,6 +423,12 @@ def run_job_thread(jid):
|
|||||||
failed_vms = []
|
failed_vms = []
|
||||||
|
|
||||||
for idx, vm in enumerate(vm_names):
|
for idx, vm in enumerate(vm_names):
|
||||||
|
if is_cancelled():
|
||||||
|
failed_vms.append((vm, "Cancelled by user"))
|
||||||
|
with open(log_path, 'a', encoding='utf-8') as f:
|
||||||
|
f.write(f"\nSkipping VM {idx+1}/{total_vms} ({vm}): Backup cancelled by user\n")
|
||||||
|
continue
|
||||||
|
|
||||||
vm_pct_start = int((idx / total_vms) * 100)
|
vm_pct_start = int((idx / total_vms) * 100)
|
||||||
vm_pct_end = int(((idx + 1) / total_vms) * 100)
|
vm_pct_end = int(((idx + 1) / total_vms) * 100)
|
||||||
|
|
||||||
@ -465,6 +473,7 @@ def run_job_thread(jid):
|
|||||||
progress_cb=make_vm_progress_cb(vm, vm_pct_start, vm_pct_end, idx, total_vms),
|
progress_cb=make_vm_progress_cb(vm, vm_pct_start, vm_pct_end, idx, total_vms),
|
||||||
disk_filter=disk_filter,
|
disk_filter=disk_filter,
|
||||||
job_id=jid,
|
job_id=jid,
|
||||||
|
is_cancelled_cb=is_cancelled,
|
||||||
)
|
)
|
||||||
success_vms.append(vm)
|
success_vms.append(vm)
|
||||||
|
|
||||||
@ -477,6 +486,13 @@ def run_job_thread(jid):
|
|||||||
}
|
}
|
||||||
enforce_retention_policy(vm_info, log_path=log_path)
|
enforce_retention_policy(vm_info, log_path=log_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if "cancelled by user" in str(e).lower():
|
||||||
|
failed_vms.append((vm, "Cancelled by user"))
|
||||||
|
info['status'] = 'failed (Cancelled)'
|
||||||
|
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
|
||||||
|
save_jobs_db()
|
||||||
|
break
|
||||||
|
else:
|
||||||
failed_vms.append((vm, str(e)))
|
failed_vms.append((vm, str(e)))
|
||||||
with open(log_path, 'a', encoding='utf-8') as f:
|
with open(log_path, 'a', encoding='utf-8') as f:
|
||||||
f.write(f"\nERROR backing up VM {vm}: {e}\n\n")
|
f.write(f"\nERROR backing up VM {vm}: {e}\n\n")
|
||||||
@ -523,6 +539,7 @@ def run_job_thread(jid):
|
|||||||
progress_cb=progress_cb,
|
progress_cb=progress_cb,
|
||||||
disk_filter=info.get('disk_filter'), # None = all disks
|
disk_filter=info.get('disk_filter'), # None = all disks
|
||||||
job_id=jid,
|
job_id=jid,
|
||||||
|
is_cancelled_cb=is_cancelled,
|
||||||
)
|
)
|
||||||
info['status'] = 'finished'
|
info['status'] = 'finished'
|
||||||
info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'}
|
info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'}
|
||||||
@ -531,6 +548,10 @@ def run_job_thread(jid):
|
|||||||
# Enforce retention policy
|
# Enforce retention policy
|
||||||
enforce_retention_policy(info, log_path=log_path)
|
enforce_retention_policy(info, log_path=log_path)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if "cancelled by user" in str(e).lower():
|
||||||
|
info['status'] = 'failed (Cancelled)'
|
||||||
|
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
|
||||||
|
else:
|
||||||
info['status'] = f'failed ({e})'
|
info['status'] = f'failed ({e})'
|
||||||
save_jobs_db()
|
save_jobs_db()
|
||||||
|
|
||||||
@ -1023,6 +1044,22 @@ def run_job_now(jobid):
|
|||||||
return redirect(url_for('job_detail', jobid=jobid))
|
return redirect(url_for('job_detail', jobid=jobid))
|
||||||
|
|
||||||
|
|
||||||
|
@app.route('/job/<jobid>/stop', methods=['POST'])
|
||||||
|
@login_required
|
||||||
|
def stop_job(jobid):
|
||||||
|
info = jobs.get(jobid)
|
||||||
|
if not info:
|
||||||
|
abort(404)
|
||||||
|
if info.get('status') in ('running', 'queued'):
|
||||||
|
info['status'] = 'cancelling'
|
||||||
|
info['progress'] = {'pct': info.get('progress', {}).get('pct', 0), 'phase': 'cancelling', 'detail': 'Stopping backup execution…'}
|
||||||
|
save_jobs_db()
|
||||||
|
flash('Request to stop backup sent.', 'info')
|
||||||
|
else:
|
||||||
|
flash('Job is not running or queued.', 'warning')
|
||||||
|
return redirect(url_for('job_detail', jobid=jobid))
|
||||||
|
|
||||||
|
|
||||||
@app.route('/job/<jobid>/delete', methods=['POST'])
|
@app.route('/job/<jobid>/delete', methods=['POST'])
|
||||||
@login_required
|
@login_required
|
||||||
def delete_job(jobid):
|
def delete_job(jobid):
|
||||||
|
|||||||
@ -159,6 +159,16 @@
|
|||||||
</button>
|
</button>
|
||||||
</form>
|
</form>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% if job.status == 'running' or job.status == 'queued' %}
|
||||||
|
<form method="post" action="/job/{{ job.id }}/stop"
|
||||||
|
style="margin: 0;"
|
||||||
|
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
|
||||||
|
<button class="btn btn-danger btn-sm" type="submit">
|
||||||
|
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
|
||||||
|
Force Stop
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
{% endif %}
|
||||||
<a href="/jobs" class="btn btn-ghost btn-sm">
|
<a href="/jobs" class="btn btn-ghost btn-sm">
|
||||||
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 6px;"><line x1="19" y1="12" x2="5" y2="12"/><polyline points="12 19 5 12 12 5"/></svg>
|
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 6px;"><line x1="19" y1="12" x2="5" y2="12"/><polyline points="12 19 5 12 12 5"/></svg>
|
||||||
All Jobs
|
All Jobs
|
||||||
|
|||||||
@ -174,6 +174,16 @@
|
|||||||
<button class="btn btn-primary btn-sm" type="submit">Run Now</button>
|
<button class="btn btn-primary btn-sm" type="submit">Run Now</button>
|
||||||
</form>
|
</form>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
{% if job.status == 'running' or job.status == 'queued' %}
|
||||||
|
<form method="post" action="/job/{{ job.id }}/stop"
|
||||||
|
style="margin: 0;"
|
||||||
|
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
|
||||||
|
<button class="btn btn-danger btn-sm" type="submit">
|
||||||
|
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
|
||||||
|
Force Stop
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
{% endif %}
|
||||||
{% if job.schedule_id %}
|
{% if job.schedule_id %}
|
||||||
<form method="post" action="/job/{{ job.id }}/cancel-schedule"
|
<form method="post" action="/job/{{ job.id }}/cancel-schedule"
|
||||||
style="margin: 0;"
|
style="margin: 0;"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user