Built the Sentinel Tab
This commit is contained in:
56
monitor_nvme.py
Normal file
56
monitor_nvme.py
Normal file
@@ -0,0 +1,56 @@
|
||||
import subprocess
|
||||
import json
|
||||
import time
|
||||
import os
|
||||
|
||||
instance_id = "i-0e4eab5fe67cf9abe"
|
||||
|
||||
def run_cmd():
|
||||
# Sends a bash command to check NVMe folder size and the transfer log
|
||||
bash_script = "du -sh /opt/dlami/nvme/models/Qwen-Image-2512 2>/dev/null; echo ''; tail -n 8 /home/ubuntu/qwen_nvme_transfer.log"
|
||||
|
||||
send_req = subprocess.run([
|
||||
"aws", "ssm", "send-command",
|
||||
"--instance-ids", instance_id,
|
||||
"--document-name", "AWS-RunShellScript",
|
||||
"--parameters", 'commands=["' + bash_script + '"]',
|
||||
"--output", "json"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
try:
|
||||
cmd_id = json.loads(send_req.stdout)["Command"]["CommandId"]
|
||||
except Exception as e:
|
||||
return f"Failed to send SSM command: {send_req.stderr}"
|
||||
|
||||
time.sleep(4) # Wait to allow the command to execute on AWS
|
||||
|
||||
res = subprocess.run([
|
||||
"aws", "ssm", "get-command-invocation",
|
||||
"--command-id", cmd_id,
|
||||
"--instance-id", instance_id,
|
||||
"--output", "json"
|
||||
], capture_output=True, text=True)
|
||||
|
||||
try:
|
||||
out = json.loads(res.stdout)
|
||||
if out["Status"] in ["Pending", "InProgress"]:
|
||||
return "Command still executing. Will retry next loop..."
|
||||
raw_out = out.get("StandardOutputContent", "") + out.get("StandardErrorContent", "")
|
||||
# CRITICAL FIX: Safe conversion for Windows CMD to prevent tqdm progress bar crash
|
||||
return raw_out.encode('utf-8', 'ignore').decode('cp1252', 'ignore')
|
||||
except:
|
||||
return "Waiting for AWS to return output..."
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Initializing AWS SSM connection for NVMe Transfer...")
|
||||
while True:
|
||||
output = run_cmd()
|
||||
os.system('cls' if os.name == 'nt' else 'clear')
|
||||
print("======== LIVE NVMe TRANSFER & EXTRACT MONITOR ========")
|
||||
print("Model: Qwen-Image-2512")
|
||||
print("Target: /opt/dlami/nvme/models/")
|
||||
print("======================================================\n")
|
||||
print(output.strip())
|
||||
print("\n======================================================")
|
||||
print("Refreshing every 10 seconds... (Press Ctrl+C to stop)")
|
||||
time.sleep(10)
|
||||
Reference in New Issue
Block a user