diff --git a/vsphere_backup/__pycache__/backup_core.cpython-310.pyc b/vsphere_backup/__pycache__/backup_core.cpython-310.pyc index 9b3a318..5efcc76 100644 Binary files a/vsphere_backup/__pycache__/backup_core.cpython-310.pyc and b/vsphere_backup/__pycache__/backup_core.cpython-310.pyc differ diff --git a/vsphere_backup/__pycache__/gui_app.cpython-310.pyc b/vsphere_backup/__pycache__/gui_app.cpython-310.pyc index d319e55..1dcbf16 100644 Binary files a/vsphere_backup/__pycache__/gui_app.cpython-310.pyc and b/vsphere_backup/__pycache__/gui_app.cpython-310.pyc differ diff --git a/vsphere_backup/backup_core.py b/vsphere_backup/backup_core.py index 2e2c530..035d8c2 100644 --- a/vsphere_backup/backup_core.py +++ b/vsphere_backup/backup_core.py @@ -113,19 +113,27 @@ def find_datacenter_for_datastore(content, datastore_name): return None -def download_datastore_file(host, dc_name, datastore_name, ds_path, local_path, session_cookie, verify_ssl=True): +def download_datastore_file(host, dc_name, datastore_name, ds_path, local_path, + session_cookie, verify_ssl=True, progress_cb=None): + """Download a file from a vSphere datastore. progress_cb(bytes_done, bytes_total) is optional.""" encoded_path = urllib.parse.quote(ds_path, safe='') - url = f"https://{host}/folder/{encoded_path}?dcPath={urllib.parse.quote(dc_name)}&dsName={urllib.parse.quote(datastore_name)}" + url = (f"https://{host}/folder/{encoded_path}" + f"?dcPath={urllib.parse.quote(dc_name)}&dsName={urllib.parse.quote(datastore_name)}") headers = {"Cookie": f"vmware_soap_session={session_cookie}"} print(f"Downloading {ds_path} from datastore {datastore_name} to {local_path}") with requests.get(url, headers=headers, stream=True, verify=verify_ssl) as r: r.raise_for_status() + total_bytes = int(r.headers.get('Content-Length', 0)) + done_bytes = 0 os.makedirs(os.path.dirname(local_path), exist_ok=True) with open(local_path, 'wb') as f: - for chunk in r.iter_content(chunk_size=10 * 1024 * 1024): + for chunk in r.iter_content(chunk_size=4 * 1024 * 1024): if chunk: f.write(chunk) - print("Download completed") + done_bytes += len(chunk) + if progress_cb: + progress_cb(done_bytes, total_bytes) + print(f"Download completed ({done_bytes // (1024*1024)} MB)") def extract_session_cookie(si): @@ -221,31 +229,42 @@ def upload_via_sftp(host, user, password, key_filename, local_path, remote_dir): def run_backup(host, user, password, vm_name, dest, compress=False, no_verify_ssl=False, - sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None, log_path=None): - """Run full backup flow. If log_path is provided, stdout/stderr will be redirected there.""" + sftp_host=None, sftp_user=None, sftp_password=None, sftp_key=None, + log_path=None, progress_cb=None): + """Run full backup flow. progress_cb(phase, pct, detail) is called with live status updates.""" if log_path: logfile = open(log_path, 'ab') - # use binary logfile; redirect prints into it def _wrap(): with redirect_stdout(logfile), redirect_stderr(logfile): return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, - sftp_host, sftp_user, sftp_password, sftp_key) + sftp_host, sftp_user, sftp_password, sftp_key, + progress_cb=progress_cb) try: return _wrap() finally: logfile.close() else: return _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, - sftp_host, sftp_user, sftp_password, sftp_key) + sftp_host, sftp_user, sftp_password, sftp_key, + progress_cb=progress_cb) def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ssl, - sftp_host, sftp_user, sftp_password, sftp_key): + sftp_host, sftp_user, sftp_password, sftp_key, progress_cb=None): + def _prog(phase, pct, detail=''): + if progress_cb: + try: + progress_cb({'phase': phase, 'pct': pct, 'detail': detail}) + except Exception: + pass + si = None try: + _prog('connecting', 0, 'Connecting to vCenter…') si = get_si(host, user, password, no_verify_ssl=no_verify_ssl) content = si.RetrieveContent() - # find vm + + _prog('connecting', 2, f'Looking up VM: {vm_name}') obj_view = content.viewManager.CreateContainerView(content.rootFolder, [vim.VirtualMachine], True) vm = None for v in obj_view.view: @@ -259,8 +278,10 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss snap_name = f"backup-{int(time.time())}" created_snapshot = False try: + _prog('snapshot', 3, 'Creating snapshot…') create_snapshot(vm, snap_name, desc="Automated backup snapshot", memory=False, quiesce=False) created_snapshot = True + _prog('snapshot', 5, 'Snapshot created') session_cookie = extract_session_cookie(si) if not session_cookie: @@ -272,8 +293,14 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss if vmx_ref: all_refs.append(vmx_ref) + total_files = len(all_refs) + # Download phase: 5% -> 90% + DOWNLOAD_START = 5 + DOWNLOAD_END = 90 + download_range = DOWNLOAD_END - DOWNLOAD_START + downloaded_files = [] - for ref in all_refs: + for file_idx, ref in enumerate(all_refs): ds_name, ds_path = parse_datastore_path(ref) dc = find_datacenter_for_datastore(content, ds_name) if not dc: @@ -281,12 +308,41 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss dc_name = dc.name safe_path = ds_path.replace('/', os.sep) local_file = os.path.join(dest, ds_name, safe_path) - download_datastore_file(host, dc_name, ds_name, ds_path, local_file, session_cookie, verify_ssl=not no_verify_ssl) + + file_base_pct = DOWNLOAD_START + int((file_idx / total_files) * download_range) + file_share = download_range / total_files + + def make_dl_cb(fidx, total, base_pct, share, fname): + def _dl_cb(done, total_b): + if total_b > 0: + file_pct = done / total_b + overall_pct = int(base_pct + file_pct * share) + done_mb = done // (1024 * 1024) + total_mb = total_b // (1024 * 1024) + detail = (f'File {fidx+1}/{total}: {fname} — ' + f'{done_mb} / {total_mb} MB ' + f'({int(file_pct*100)}%)') + else: + overall_pct = base_pct + detail = f'File {fidx+1}/{total}: {fname}' + _prog('downloading', overall_pct, detail) + return _dl_cb + + _prog('downloading', file_base_pct, + f'Starting file {file_idx+1}/{total_files}: {os.path.basename(ds_path)}') + download_datastore_file( + host, dc_name, ds_name, ds_path, local_file, session_cookie, + verify_ssl=not no_verify_ssl, + progress_cb=make_dl_cb(file_idx, total_files, file_base_pct, + file_share, os.path.basename(ds_path)) + ) downloaded_files.append(local_file) + _prog('compressing', 90, 'Download complete') final_files = [] for f in downloaded_files: if compress: + _prog('compressing', 92, f'Compressing {os.path.basename(f)}…') cf = maybe_compress(f) final_files.append(cf) else: @@ -295,9 +351,11 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss if sftp_host: if not sftp_user: raise Exception('SFTP user required') + _prog('uploading', 95, f'Uploading to {sftp_host}…') for f in final_files: upload_via_sftp(sftp_host, sftp_user, sftp_password, sftp_key, f, os.path.basename(dest)) + _prog('cleanup', 97, 'Removing snapshot…') print('Backup completed successfully') finally: if created_snapshot: @@ -311,6 +369,7 @@ def _run_backup_impl(host, user, password, vm_name, dest, compress, no_verify_ss print(f'Failed to remove snapshot: {e}', file=sys.stderr) else: print('Snapshot object not found in tree; may have been removed already') + _prog('done', 100, 'Backup finished successfully') finally: if si: try: diff --git a/vsphere_backup/gui_app.py b/vsphere_backup/gui_app.py index b9c3737..170de2a 100644 --- a/vsphere_backup/gui_app.py +++ b/vsphere_backup/gui_app.py @@ -7,6 +7,8 @@ import sys import uuid import threading import time +import platform +import subprocess import json from datetime import datetime from functools import wraps @@ -19,6 +21,8 @@ from flask import ( from backup_core import run_backup, list_vms +IS_LINUX = platform.system() == 'Linux' + # ── APScheduler (optional graceful degradation) ────────────────────────────── try: from apscheduler.schedulers.background import BackgroundScheduler @@ -50,6 +54,137 @@ if HAS_SCHEDULER: scheduler = BackgroundScheduler(daemon=True) scheduler.start() +# ── VM list cache ───────────────────────────────────────────────────────────── +# Keyed by (host, user) so different users get separate caches. +_vm_cache: dict = {} # key -> {'vms': [...], 'ts': float, 'error': str|None} +_vm_cache_lock = threading.Lock() +VM_CACHE_TTL = 60 # seconds before background refresh + + +def _cache_key(host, user): + return f'{host}::{user}' + + +def get_cached_vms(host, user, password, no_verify_ssl=False, force=False): + """ + Return VM list from cache. If cache is missing or expired, fetch synchronously. + A background thread keeps the cache warm after the first fetch. + """ + key = _cache_key(host, user) + with _vm_cache_lock: + entry = _vm_cache.get(key) + + now = time.time() + if not force and entry and (now - entry['ts']) < VM_CACHE_TTL: + # Fresh cache — return immediately + return entry['vms'], entry['error'], entry['ts'] + + if not force and entry: + # Stale but exists — return stale data and kick off background refresh + _start_bg_refresh(host, user, password, no_verify_ssl) + return entry['vms'], entry['error'], entry['ts'] + + # No cache at all — must fetch synchronously (first load or forced refresh) + return _fetch_and_cache(host, user, password, no_verify_ssl) + + +def _fetch_and_cache(host, user, password, no_verify_ssl): + """Fetch VM list from vSphere and store in cache. Returns (vms, error, ts).""" + key = _cache_key(host, user) + try: + vms = list_vms(host, user, password, no_verify_ssl=no_verify_ssl) + order = {'poweredOn': 0, 'suspended': 1, 'poweredOff': 2} + vms.sort(key=lambda v: (order.get(v['power_state'], 3), v['name'].lower())) + entry = {'vms': vms, 'ts': time.time(), 'error': None} + except Exception as e: + # Keep old VM list on error, just update error message + with _vm_cache_lock: + old = _vm_cache.get(key, {}) + entry = {'vms': old.get('vms', []), 'ts': time.time(), 'error': str(e)} + with _vm_cache_lock: + _vm_cache[key] = entry + return entry['vms'], entry['error'], entry['ts'] + + +_bg_refresh_running: set = set() + + +def _start_bg_refresh(host, user, password, no_verify_ssl): + """Kick off a background thread to refresh the cache if not already running.""" + key = _cache_key(host, user) + if key in _bg_refresh_running: + return + _bg_refresh_running.add(key) + + def _worker(): + try: + _fetch_and_cache(host, user, password, no_verify_ssl) + finally: + _bg_refresh_running.discard(key) + + t = threading.Thread(target=_worker, daemon=True) + t.start() + + +# ── NFS management (Linux only) ───────────────────────────────────────────────── + +def list_nfs_mounts(): + """Return list of currently mounted NFS/CIFS shares from /proc/mounts.""" + mounts = [] + if not IS_LINUX: + return mounts + try: + with open('/proc/mounts', 'r') as f: + for line in f: + parts = line.strip().split() + if len(parts) >= 4 and parts[2] in ('nfs', 'nfs4', 'cifs'): + info = {'device': parts[0], 'mountpoint': parts[1], + 'fstype': parts[2], 'options': parts[3]} + # Add disk space info + try: + st = os.statvfs(parts[1]) + total = st.f_blocks * st.f_frsize + free = st.f_available * st.f_frsize + used = total - free + info['total_gb'] = round(total / (1024**3), 1) + info['used_gb'] = round(used / (1024**3), 1) + info['free_gb'] = round(free / (1024**3), 1) + info['pct_used'] = int(used / total * 100) if total > 0 else 0 + except Exception: + info.update({'total_gb': 0, 'used_gb': 0, 'free_gb': 0, 'pct_used': 0}) + mounts.append(info) + except (FileNotFoundError, PermissionError): + pass + return mounts + + +def mount_nfs(server, export, mountpoint, nfs_version='4', extra_opts=''): + """Mount an NFS share (Linux only).""" + if not IS_LINUX: + raise RuntimeError('NFS mounting is only supported on Linux') + os.makedirs(mountpoint, exist_ok=True) + opts = [] + if nfs_version: + opts.append(f'vers={nfs_version}') + if extra_opts: + opts.append(extra_opts.strip()) + cmd = ['mount', '-t', 'nfs'] + if opts: + cmd += ['-o', ','.join(opts)] + cmd += [f'{server}:{export}', mountpoint] + result = subprocess.run(cmd, capture_output=True, text=True, timeout=30) + if result.returncode != 0: + raise RuntimeError((result.stderr or result.stdout or 'mount failed').strip()) + + +def umount_nfs(mountpoint): + """Unmount an NFS share (Linux only).""" + if not IS_LINUX: + raise RuntimeError('NFS unmounting is only supported on Linux') + result = subprocess.run(['umount', mountpoint], capture_output=True, text=True, timeout=30) + if result.returncode != 0: + raise RuntimeError((result.stderr or result.stdout or 'umount failed').strip()) + # ── Helpers ─────────────────────────────────────────────────────────────────── def login_required(f): @@ -88,9 +223,14 @@ def run_job_thread(jid): info = jobs.get(jid) if not info: return - info['status'] = 'running' - info['started'] = time.time() + info['status'] = 'running' + info['started'] = time.time() + info['progress'] = {'pct': 0, 'phase': 'starting', 'detail': 'Initializing…'} log_path = str(JOBS_DIR / jid / 'backup.log') + + def progress_cb(prog): + info['progress'] = prog + try: run_backup( host=info['host'], @@ -105,8 +245,10 @@ def run_job_thread(jid): sftp_password=info.get('sftp_password') or None, sftp_key=None, log_path=log_path, + progress_cb=progress_cb, ) - info['status'] = 'finished' + info['status'] = 'finished' + info['progress'] = {'pct': 100, 'phase': 'done', 'detail': 'Backup completed successfully'} except Exception as e: info['status'] = f'failed ({e})' @@ -200,27 +342,26 @@ def index(): @app.route('/login', methods=['GET', 'POST']) def login(): if request.method == 'POST': - host = request.form.get('host', '').strip() - user = request.form.get('user', '').strip() - password = request.form.get('password', '') + host = request.form.get('host', '').strip() + user = request.form.get('user', '').strip() + password = request.form.get('password', '') no_verify_ssl = 'no_verify_ssl' in request.form if not (host and user and password): flash('Host, username and password are required.', 'danger') return render_template('login.html') - # Verify credentials by listing VMs - try: - list_vms(host, user, password, no_verify_ssl=no_verify_ssl) - except Exception as e: - flash(f'Connection failed: {e}', 'danger') + # Verify credentials — also warms the VM cache for instant /vms load + vm_list, error, _ = _fetch_and_cache(host, user, password, no_verify_ssl) + if error and not vm_list: + flash(f'Connection failed: {error}', 'danger') return render_template('login.html') session['host'] = host session['user'] = user session['password'] = password session['no_verify_ssl'] = no_verify_ssl - flash(f'Connected to {host} successfully.', 'success') + flash(f'Connected to {host} — {len(vm_list)} VMs found.', 'success') return redirect(url_for('vms')) return render_template('login.html') @@ -238,32 +379,28 @@ def logout(): @app.route('/vms') @login_required def vms(): - error = None - vm_list = [] - try: - vm_list = list_vms( - session['host'], session['user'], session['password'], - no_verify_ssl=session.get('no_verify_ssl', False) - ) - # Sort: powered on first, then alphabetical - order = {'poweredOn': 0, 'suspended': 1, 'poweredOff': 2} - vm_list.sort(key=lambda v: (order.get(v['power_state'], 3), v['name'].lower())) - except Exception as e: - error = str(e) - return render_template('vms.html', vms=vm_list, error=error) + force = request.args.get('refresh') == '1' + vm_list, error, cache_ts = get_cached_vms( + session['host'], session['user'], session['password'], + no_verify_ssl=session.get('no_verify_ssl', False), + force=force, + ) + cache_age = int(time.time() - cache_ts) if cache_ts else None + return render_template('vms.html', vms=vm_list, error=error, cache_age=cache_age) @app.route('/api/vms') @login_required def api_vms(): - try: - vm_list = list_vms( - session['host'], session['user'], session['password'], - no_verify_ssl=session.get('no_verify_ssl', False) - ) - return jsonify(vm_list) - except Exception as e: - return jsonify({'error': str(e)}), 500 + force = request.args.get('refresh') == '1' + vm_list, error, cache_ts = get_cached_vms( + session['host'], session['user'], session['password'], + no_verify_ssl=session.get('no_verify_ssl', False), + force=force, + ) + if error and not vm_list: + return jsonify({'error': error}), 500 + return jsonify({'vms': vm_list, 'cache_age': int(time.time() - cache_ts) if cache_ts else None}) # ── Create Job ──────────────────────────────────────────────────────────────── @@ -316,17 +453,16 @@ def create_job(): return redirect(url_for('job_detail', jobid=jid)) # GET: load VM list for the dropdown - selected_vm = request.args.get('vm', '') + selected_vm = request.args.get('vm', '') show_schedule = bool(request.args.get('schedule', '')) - vm_list = [] - try: - vm_list = list_vms( - session['host'], session['user'], session['password'], - no_verify_ssl=session.get('no_verify_ssl', False) - ) - vm_list.sort(key=lambda v: v['name'].lower()) - except Exception as e: - flash(f'Could not load VM list: {e}', 'danger') + vm_list, error, _ = get_cached_vms( + session['host'], session['user'], session['password'], + no_verify_ssl=session.get('no_verify_ssl', False) + ) + if error and not vm_list: + flash(f'Could not load VM list: {error}', 'danger') + # Sort alphabetically for the dropdown + vm_list = sorted(vm_list, key=lambda v: v['name'].lower()) return render_template( 'create_job.html', @@ -380,7 +516,11 @@ def api_job_status(jobid): info = jobs.get(jobid) if not info: return jsonify({'error': 'not found'}), 404 - return jsonify({'status': info.get('status', 'unknown'), 'id': jobid}) + return jsonify({ + 'status': info.get('status', 'unknown'), + 'id': jobid, + 'progress': info.get('progress', {'pct': 0, 'phase': '', 'detail': ''}), + }) @app.route('/job//cancel-schedule', methods=['POST']) @@ -407,6 +547,56 @@ def startswith_filter(value, prefix): return str(value).startswith(prefix) +# ── NFS Management Routes ───────────────────────────────────────────────────────── + +@app.route('/nfs') +@login_required +def nfs_manager(): + mounts = list_nfs_mounts() + return render_template('nfs.html', mounts=mounts, is_linux=IS_LINUX) + + +@app.route('/nfs/mount', methods=['POST']) +@login_required +def nfs_mount(): + server = request.form.get('server', '').strip() + export = request.form.get('export', '').strip() + mountpoint = request.form.get('mountpoint', '').strip() + nfs_ver = request.form.get('nfs_version', '4') + extra_opts = request.form.get('extra_opts', '').strip() + + if not (server and export and mountpoint): + flash('Server, export path, and mount point are required.', 'danger') + return redirect(url_for('nfs_manager')) + try: + mount_nfs(server, export, mountpoint, nfs_version=nfs_ver, extra_opts=extra_opts) + flash(f'Mounted {server}:{export} → {mountpoint} successfully.', 'success') + except Exception as e: + flash(f'Mount failed: {e}', 'danger') + return redirect(url_for('nfs_manager')) + + +@app.route('/nfs/umount', methods=['POST']) +@login_required +def nfs_umount(): + mountpoint = request.form.get('mountpoint', '').strip() + if not mountpoint: + flash('Mount point is required.', 'danger') + return redirect(url_for('nfs_manager')) + try: + umount_nfs(mountpoint) + flash(f'Unmounted {mountpoint} successfully.', 'success') + except Exception as e: + flash(f'Unmount failed: {e}', 'danger') + return redirect(url_for('nfs_manager')) + + +@app.route('/api/nfs') +@login_required +def api_nfs(): + return jsonify(list_nfs_mounts()) + + # ── Main ────────────────────────────────────────────────────────────────────── if __name__ == '__main__': app.run(host='0.0.0.0', port=5000, debug=False) diff --git a/vsphere_backup/templates/base.html b/vsphere_backup/templates/base.html index d3b5da3..3bdf656 100644 --- a/vsphere_backup/templates/base.html +++ b/vsphere_backup/templates/base.html @@ -299,6 +299,9 @@ Create Job + + 📡 NFS Manager +