feat: implement backup cancellation support and add job management UI templates

This commit is contained in:
Rizqi 2026-06-22 02:49:24 +07:00
parent 04fb84c44a
commit f6f6158811
4 changed files with 80 additions and 8 deletions

View File

@ -327,7 +327,8 @@ def upload_via_sftp(host, user, password, key_filename, local_path, remote_dir):
def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False, def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False,
sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None, sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None,
log_path=None, progress_cb=None, disk_filter=None, job_id=None): log_path=None, progress_cb=None, disk_filter=None, job_id=None,
is_cancelled_cb=None):
"""Run full backup flow. """Run full backup flow.
disk_filter: if not None, a set/list of VMDK file-ref strings to include. disk_filter: if not None, a set/list of VMDK file-ref strings to include.
The VMX config file is always included regardless. The VMX config file is always included regardless.
@ -338,7 +339,8 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
with redirect_stdout(logfile), redirect_stderr(logfile): with redirect_stdout(logfile), redirect_stderr(logfile):
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
sftp_host, sftp_user, sftp_password, sftp_key, sftp_host, sftp_user, sftp_password, sftp_key,
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id) progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
is_cancelled_cb=is_cancelled_cb)
try: try:
return _wrap() return _wrap()
finally: finally:
@ -346,12 +348,14 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
else: else:
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
sftp_host, sftp_user, sftp_password, sftp_key, sftp_host, sftp_user, sftp_password, sftp_key,
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id) progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
is_cancelled_cb=is_cancelled_cb)
def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
sftp_host, sftp_user, sftp_password, sftp_key, sftp_host, sftp_user, sftp_password, sftp_key,
progress_cb=None, disk_filter=None, job_id=None): progress_cb=None, disk_filter=None, job_id=None,
is_cancelled_cb=None):
def _prog(phase, pct, detail=''): def _prog(phase, pct, detail=''):
if progress_cb: if progress_cb:
try: try:
@ -428,6 +432,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
downloaded_files = [] downloaded_files = []
files_manifest_info = [] files_manifest_info = []
for file_idx, ref in enumerate(all_refs): for file_idx, ref in enumerate(all_refs):
if is_cancelled_cb and is_cancelled_cb():
raise RuntimeError("Backup cancelled by user")
ds_name, ds_path = parse_datastore_path(ref) ds_name, ds_path = parse_datastore_path(ref)
dc = find_datacenter_for_datastore(content, ds_name) dc = find_datacenter_for_datastore(content, ds_name)
if not dc: if not dc:
@ -441,6 +447,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
def make_dl_cb(fidx, total, base_pct, share, fname): def make_dl_cb(fidx, total, base_pct, share, fname):
def _dl_cb(done, total_b): def _dl_cb(done, total_b):
if is_cancelled_cb and is_cancelled_cb():
raise RuntimeError("Backup cancelled by user")
if total_b > 0: if total_b > 0:
file_pct = done / total_b file_pct = done / total_b
overall_pct = int(base_pct + file_pct * share) overall_pct = int(base_pct + file_pct * share)
@ -482,6 +490,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
}) })
_prog('compressing', 90, 'Downloads complete. Creating manifest…') _prog('compressing', 90, 'Downloads complete. Creating manifest…')
if is_cancelled_cb and is_cancelled_cb():
raise RuntimeError("Backup cancelled by user")
# Write manifest.json # Write manifest.json
finished_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') finished_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
@ -501,6 +511,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
final_files = [] final_files = []
for f in downloaded_files: for f in downloaded_files:
if is_cancelled_cb and is_cancelled_cb():
raise RuntimeError("Backup cancelled by user")
if compress: if compress:
_prog('compressing', 92, f'Compressing {os.path.basename(f)}') _prog('compressing', 92, f'Compressing {os.path.basename(f)}')
cf = maybe_compress(f) cf = maybe_compress(f)
@ -515,6 +527,9 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
if not sftp_user: if not sftp_user:
raise Exception('SFTP user required') raise Exception('SFTP user required')
if is_cancelled_cb and is_cancelled_cb():
raise RuntimeError("Backup cancelled by user")
# Verify checksums before upload # Verify checksums before upload
_prog('uploading', 94, 'Verifying local checksums before SFTP upload…') _prog('uploading', 94, 'Verifying local checksums before SFTP upload…')
print("Running pre-upload checksum verification...") print("Running pre-upload checksum verification...")

View File

@ -408,6 +408,8 @@ def run_job_thread(jid):
info['started'] = time.time() info['started'] = time.time()
info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'} info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'}
is_cancelled = lambda: jobs.get(jid, {}).get('status') == 'cancelling'
vm_names = info.get('vm_names') vm_names = info.get('vm_names')
log_path = str(JOBS_DIR / jid / 'backup.log') log_path = str(JOBS_DIR / jid / 'backup.log')
@ -421,6 +423,12 @@ def run_job_thread(jid):
failed_vms = [] failed_vms = []
for idx, vm in enumerate(vm_names): for idx, vm in enumerate(vm_names):
if is_cancelled():
failed_vms.append((vm, "Cancelled by user"))
with open(log_path, 'a', encoding='utf-8') as f:
f.write(f"\nSkipping VM {idx+1}/{total_vms} ({vm}): Backup cancelled by user\n")
continue
vm_pct_start = int((idx / total_vms) * 100) vm_pct_start = int((idx / total_vms) * 100)
vm_pct_end = int(((idx + 1) / total_vms) * 100) vm_pct_end = int(((idx + 1) / total_vms) * 100)
@ -465,6 +473,7 @@ def run_job_thread(jid):
progress_cb=make_vm_progress_cb(vm, vm_pct_start, vm_pct_end, idx, total_vms), progress_cb=make_vm_progress_cb(vm, vm_pct_start, vm_pct_end, idx, total_vms),
disk_filter=disk_filter, disk_filter=disk_filter,
job_id=jid, job_id=jid,
is_cancelled_cb=is_cancelled,
) )
success_vms.append(vm) success_vms.append(vm)
@ -477,9 +486,16 @@ def run_job_thread(jid):
} }
enforce_retention_policy(vm_info, log_path=log_path) enforce_retention_policy(vm_info, log_path=log_path)
except Exception as e: except Exception as e:
failed_vms.append((vm, str(e))) if "cancelled by user" in str(e).lower():
with open(log_path, 'a', encoding='utf-8') as f: failed_vms.append((vm, "Cancelled by user"))
f.write(f"\nERROR backing up VM {vm}: {e}\n\n") info['status'] = 'failed (Cancelled)'
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
save_jobs_db()
break
else:
failed_vms.append((vm, str(e)))
with open(log_path, 'a', encoding='utf-8') as f:
f.write(f"\nERROR backing up VM {vm}: {e}\n\n")
if failed_vms: if failed_vms:
if success_vms: if success_vms:
@ -523,6 +539,7 @@ def run_job_thread(jid):
progress_cb=progress_cb, progress_cb=progress_cb,
disk_filter=info.get('disk_filter'), # None = all disks disk_filter=info.get('disk_filter'), # None = all disks
job_id=jid, job_id=jid,
is_cancelled_cb=is_cancelled,
) )
info['status'] = 'finished' info['status'] = 'finished'
info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'} info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'}
@ -531,7 +548,11 @@ def run_job_thread(jid):
# Enforce retention policy # Enforce retention policy
enforce_retention_policy(info, log_path=log_path) enforce_retention_policy(info, log_path=log_path)
except Exception as e: except Exception as e:
info['status'] = f'failed ({e})' if "cancelled by user" in str(e).lower():
info['status'] = 'failed (Cancelled)'
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
else:
info['status'] = f'failed ({e})'
save_jobs_db() save_jobs_db()
@ -1023,6 +1044,22 @@ def run_job_now(jobid):
return redirect(url_for('job_detail', jobid=jobid)) return redirect(url_for('job_detail', jobid=jobid))
@app.route('/job/<jobid>/stop', methods=['POST'])
@login_required
def stop_job(jobid):
info = jobs.get(jobid)
if not info:
abort(404)
if info.get('status') in ('running', 'queued'):
info['status'] = 'cancelling'
info['progress'] = {'pct': info.get('progress', {}).get('pct', 0), 'phase': 'cancelling', 'detail': 'Stopping backup execution…'}
save_jobs_db()
flash('Request to stop backup sent.', 'info')
else:
flash('Job is not running or queued.', 'warning')
return redirect(url_for('job_detail', jobid=jobid))
@app.route('/job/<jobid>/delete', methods=['POST']) @app.route('/job/<jobid>/delete', methods=['POST'])
@login_required @login_required
def delete_job(jobid): def delete_job(jobid):

View File

@ -159,6 +159,16 @@
</button> </button>
</form> </form>
{% endif %} {% endif %}
{% if job.status == 'running' or job.status == 'queued' %}
<form method="post" action="/job/{{ job.id }}/stop"
style="margin: 0;"
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
<button class="btn btn-danger btn-sm" type="submit">
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
Force Stop
</button>
</form>
{% endif %}
<a href="/jobs" class="btn btn-ghost btn-sm"> <a href="/jobs" class="btn btn-ghost btn-sm">
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 6px;"><line x1="19" y1="12" x2="5" y2="12"/><polyline points="12 19 5 12 12 5"/></svg> <svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 6px;"><line x1="19" y1="12" x2="5" y2="12"/><polyline points="12 19 5 12 12 5"/></svg>
All Jobs All Jobs

View File

@ -174,6 +174,16 @@
<button class="btn btn-primary btn-sm" type="submit">Run Now</button> <button class="btn btn-primary btn-sm" type="submit">Run Now</button>
</form> </form>
{% endif %} {% endif %}
{% if job.status == 'running' or job.status == 'queued' %}
<form method="post" action="/job/{{ job.id }}/stop"
style="margin: 0;"
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
<button class="btn btn-danger btn-sm" type="submit">
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
Force Stop
</button>
</form>
{% endif %}
{% if job.schedule_id %} {% if job.schedule_id %}
<form method="post" action="/job/{{ job.id }}/cancel-schedule" <form method="post" action="/job/{{ job.id }}/cancel-schedule"
style="margin: 0;" style="margin: 0;"