feat: implement backup cancellation support and add job management UI templates
This commit is contained in:
parent
04fb84c44a
commit
f6f6158811
@ -327,7 +327,8 @@ def upload_via_sftp(host, user, password, key_filename, local_path, remote_dir):
|
||||
|
||||
def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False,
|
||||
sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None,
|
||||
log_path=None, progress_cb=None, disk_filter=None, job_id=None):
|
||||
log_path=None, progress_cb=None, disk_filter=None, job_id=None,
|
||||
is_cancelled_cb=None):
|
||||
"""Run full backup flow.
|
||||
disk_filter: if not None, a set/list of VMDK file-ref strings to include.
|
||||
The VMX config file is always included regardless.
|
||||
@ -338,7 +339,8 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
|
||||
with redirect_stdout(logfile), redirect_stderr(logfile):
|
||||
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id)
|
||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
|
||||
is_cancelled_cb=is_cancelled_cb)
|
||||
try:
|
||||
return _wrap()
|
||||
finally:
|
||||
@ -346,12 +348,14 @@ def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ss
|
||||
else:
|
||||
return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id)
|
||||
progress_cb=progress_cb, disk_filter=disk_filter, job_id=job_id,
|
||||
is_cancelled_cb=is_cancelled_cb)
|
||||
|
||||
|
||||
def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl,
|
||||
sftp_host, sftp_user, sftp_password, sftp_key,
|
||||
progress_cb=None, disk_filter=None, job_id=None):
|
||||
progress_cb=None, disk_filter=None, job_id=None,
|
||||
is_cancelled_cb=None):
|
||||
def _prog(phase, pct, detail=''):
|
||||
if progress_cb:
|
||||
try:
|
||||
@ -428,6 +432,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
||||
downloaded_files = []
|
||||
files_manifest_info = []
|
||||
for file_idx, ref in enumerate(all_refs):
|
||||
if is_cancelled_cb and is_cancelled_cb():
|
||||
raise RuntimeError("Backup cancelled by user")
|
||||
ds_name, ds_path = parse_datastore_path(ref)
|
||||
dc = find_datacenter_for_datastore(content, ds_name)
|
||||
if not dc:
|
||||
@ -441,6 +447,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
||||
|
||||
def make_dl_cb(fidx, total, base_pct, share, fname):
|
||||
def _dl_cb(done, total_b):
|
||||
if is_cancelled_cb and is_cancelled_cb():
|
||||
raise RuntimeError("Backup cancelled by user")
|
||||
if total_b > 0:
|
||||
file_pct = done / total_b
|
||||
overall_pct = int(base_pct + file_pct * share)
|
||||
@ -482,6 +490,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
||||
})
|
||||
|
||||
_prog('compressing', 90, 'Downloads complete. Creating manifest…')
|
||||
if is_cancelled_cb and is_cancelled_cb():
|
||||
raise RuntimeError("Backup cancelled by user")
|
||||
|
||||
# Write manifest.json
|
||||
finished_iso = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
@ -501,6 +511,8 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
||||
|
||||
final_files = []
|
||||
for f in downloaded_files:
|
||||
if is_cancelled_cb and is_cancelled_cb():
|
||||
raise RuntimeError("Backup cancelled by user")
|
||||
if compress:
|
||||
_prog('compressing', 92, f'Compressing {os.path.basename(f)}…')
|
||||
cf = maybe_compress(f)
|
||||
@ -515,6 +527,9 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss
|
||||
if not sftp_user:
|
||||
raise Exception('SFTP user required')
|
||||
|
||||
if is_cancelled_cb and is_cancelled_cb():
|
||||
raise RuntimeError("Backup cancelled by user")
|
||||
|
||||
# Verify checksums before upload
|
||||
_prog('uploading', 94, 'Verifying local checksums before SFTP upload…')
|
||||
print("Running pre-upload checksum verification...")
|
||||
|
||||
37
gui_app.py
37
gui_app.py
@ -408,6 +408,8 @@ def run_job_thread(jid):
|
||||
info['started'] = time.time()
|
||||
info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'}
|
||||
|
||||
is_cancelled = lambda: jobs.get(jid, {}).get('status') == 'cancelling'
|
||||
|
||||
vm_names = info.get('vm_names')
|
||||
log_path = str(JOBS_DIR / jid / 'backup.log')
|
||||
|
||||
@ -421,6 +423,12 @@ def run_job_thread(jid):
|
||||
failed_vms = []
|
||||
|
||||
for idx, vm in enumerate(vm_names):
|
||||
if is_cancelled():
|
||||
failed_vms.append((vm, "Cancelled by user"))
|
||||
with open(log_path, 'a', encoding='utf-8') as f:
|
||||
f.write(f"\nSkipping VM {idx+1}/{total_vms} ({vm}): Backup cancelled by user\n")
|
||||
continue
|
||||
|
||||
vm_pct_start = int((idx / total_vms) * 100)
|
||||
vm_pct_end = int(((idx + 1) / total_vms) * 100)
|
||||
|
||||
@ -465,6 +473,7 @@ def run_job_thread(jid):
|
||||
progress_cb=make_vm_progress_cb(vm, vm_pct_start, vm_pct_end, idx, total_vms),
|
||||
disk_filter=disk_filter,
|
||||
job_id=jid,
|
||||
is_cancelled_cb=is_cancelled,
|
||||
)
|
||||
success_vms.append(vm)
|
||||
|
||||
@ -477,6 +486,13 @@ def run_job_thread(jid):
|
||||
}
|
||||
enforce_retention_policy(vm_info, log_path=log_path)
|
||||
except Exception as e:
|
||||
if "cancelled by user" in str(e).lower():
|
||||
failed_vms.append((vm, "Cancelled by user"))
|
||||
info['status'] = 'failed (Cancelled)'
|
||||
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
|
||||
save_jobs_db()
|
||||
break
|
||||
else:
|
||||
failed_vms.append((vm, str(e)))
|
||||
with open(log_path, 'a', encoding='utf-8') as f:
|
||||
f.write(f"\nERROR backing up VM {vm}: {e}\n\n")
|
||||
@ -523,6 +539,7 @@ def run_job_thread(jid):
|
||||
progress_cb=progress_cb,
|
||||
disk_filter=info.get('disk_filter'), # None = all disks
|
||||
job_id=jid,
|
||||
is_cancelled_cb=is_cancelled,
|
||||
)
|
||||
info['status'] = 'finished'
|
||||
info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'}
|
||||
@ -531,6 +548,10 @@ def run_job_thread(jid):
|
||||
# Enforce retention policy
|
||||
enforce_retention_policy(info, log_path=log_path)
|
||||
except Exception as e:
|
||||
if "cancelled by user" in str(e).lower():
|
||||
info['status'] = 'failed (Cancelled)'
|
||||
info['progress'] = {'pct': 100, 'phase': 'failed', 'detail': 'Backup cancelled by user'}
|
||||
else:
|
||||
info['status'] = f'failed ({e})'
|
||||
save_jobs_db()
|
||||
|
||||
@ -1023,6 +1044,22 @@ def run_job_now(jobid):
|
||||
return redirect(url_for('job_detail', jobid=jobid))
|
||||
|
||||
|
||||
@app.route('/job/<jobid>/stop', methods=['POST'])
|
||||
@login_required
|
||||
def stop_job(jobid):
|
||||
info = jobs.get(jobid)
|
||||
if not info:
|
||||
abort(404)
|
||||
if info.get('status') in ('running', 'queued'):
|
||||
info['status'] = 'cancelling'
|
||||
info['progress'] = {'pct': info.get('progress', {}).get('pct', 0), 'phase': 'cancelling', 'detail': 'Stopping backup execution…'}
|
||||
save_jobs_db()
|
||||
flash('Request to stop backup sent.', 'info')
|
||||
else:
|
||||
flash('Job is not running or queued.', 'warning')
|
||||
return redirect(url_for('job_detail', jobid=jobid))
|
||||
|
||||
|
||||
@app.route('/job/<jobid>/delete', methods=['POST'])
|
||||
@login_required
|
||||
def delete_job(jobid):
|
||||
|
||||
@ -159,6 +159,16 @@
|
||||
</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if job.status == 'running' or job.status == 'queued' %}
|
||||
<form method="post" action="/job/{{ job.id }}/stop"
|
||||
style="margin: 0;"
|
||||
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
|
||||
<button class="btn btn-danger btn-sm" type="submit">
|
||||
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
|
||||
Force Stop
|
||||
</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
<a href="/jobs" class="btn btn-ghost btn-sm">
|
||||
<svg width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 6px;"><line x1="19" y1="12" x2="5" y2="12"/><polyline points="12 19 5 12 12 5"/></svg>
|
||||
All Jobs
|
||||
|
||||
@ -174,6 +174,16 @@
|
||||
<button class="btn btn-primary btn-sm" type="submit">Run Now</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if job.status == 'running' or job.status == 'queued' %}
|
||||
<form method="post" action="/job/{{ job.id }}/stop"
|
||||
style="margin: 0;"
|
||||
onsubmit="return confirm('Are you sure you want to stop this running backup?')">
|
||||
<button class="btn btn-danger btn-sm" type="submit">
|
||||
<svg width="11" height="11" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.5" stroke-linecap="round" stroke-linejoin="round" style="margin-right: 4px;"><rect x="3" y="3" width="18" height="18" rx="2" ry="2"/></svg>
|
||||
Force Stop
|
||||
</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
{% if job.schedule_id %}
|
||||
<form method="post" action="/job/{{ job.id }}/cancel-schedule"
|
||||
style="margin: 0;"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user