mirror of
https://github.com/steveseguin/caption-local.git
synced 2026-10-10 18:19:31 +00:00
87 lines
5.1 KiB
Python
87 lines
5.1 KiB
Python
"""Own an isolated CUDA service, browser workload and NVIDIA/resource monitor."""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
import psutil
|
|
from service_test_support import TestService
|
|
from summarize_browser_load import summarize
|
|
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--model',default='medium')
|
|
parser.add_argument('--compute-type',default='float16')
|
|
parser.add_argument('--workers',type=int,default=2)
|
|
parser.add_argument('--beam-size',type=int,default=5)
|
|
parser.add_argument('--streams',type=int,default=12)
|
|
parser.add_argument('--seconds',type=int,default=180)
|
|
parser.add_argument('--interval',type=int,choices=[3,6,9],default=6)
|
|
modes=parser.add_mutually_exclusive_group()
|
|
modes.add_argument('--mixed',action='store_true')
|
|
modes.add_argument('--rotate-modes',action='store_true')
|
|
parser.add_argument('--output',type=Path,required=True)
|
|
args=parser.parse_args()
|
|
if subprocess.check_output(['nvidia-smi','--query-compute-apps=pid','--format=csv,noheader'],text=True).strip():
|
|
parser.error('GPU occupied; preserve existing workloads')
|
|
args.output.mkdir(parents=True,exist_ok=True)
|
|
stop_file=args.output/'stop-requested'
|
|
if stop_file.exists(): parser.error('Output contains a previous stop signal; choose a new output directory')
|
|
service=TestService(8772,args.output/'service.log',['--model',args.model,'--device','cuda',
|
|
'--compute-type',args.compute_type,'--workers',str(args.workers),'--threads','4',
|
|
'--beam-size',str(args.beam_size),'--max-streams',str(args.streams)])
|
|
finished=threading.Event(); samples=[]; monitor_errors=[]; thread=None
|
|
headers={'Authorization':'Bearer '+os.environ['CAPTION_API_KEY']} if os.environ.get('CAPTION_API_KEY') else {}
|
|
def health():
|
|
with urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:8772/health',headers=headers),timeout=5) as response:
|
|
return json.load(response)
|
|
def monitor():
|
|
started=time.monotonic(); process=psutil.Process(service.process.pid)
|
|
while not finished.is_set():
|
|
try:
|
|
info=health(); processes=[process]+process.children(recursive=True)
|
|
gpu=subprocess.check_output(['nvidia-smi','--query-gpu=memory.used,utilization.gpu,power.draw','--format=csv,noheader,nounits'],text=True).strip()
|
|
active=subprocess.check_output(['nvidia-smi','--query-compute-apps=pid,process_name','--format=csv,noheader'],text=True).strip().splitlines()
|
|
owned={str(p.pid) for p in processes}
|
|
foreign=[line for line in active if line.split(',')[0].strip() not in owned]
|
|
if foreign:
|
|
monitor_errors.append('Competing GPU process: '+repr(foreign))
|
|
stop_file.write_text('Competing GPU workload detected; drain capture and retain failure evidence.',encoding='utf-8')
|
|
sample={'seconds':round(time.monotonic()-started,2),
|
|
'rss_mib':round(sum(p.memory_info().rss for p in processes)/2**20,2),
|
|
'cpu_seconds':round(sum(p.cpu_times().user+p.cpu_times().system for p in processes),3),
|
|
'nvidia_smi':gpu, 'gpu_compute_processes':active, **{k:info[k] for k in ('ready','running','pending','completed','failed','rejected','queue_timeouts')}}
|
|
samples.append(sample)
|
|
(args.output/'monitor.json').write_text(json.dumps(samples,indent=2)+'\n',encoding='utf-8')
|
|
except Exception as exc:
|
|
monitor_errors.append(repr(exc))
|
|
finished.wait(5)
|
|
try:
|
|
began=time.monotonic(); service.start(); info=health()
|
|
assert info['device']=='cuda',info
|
|
(args.output/'configuration.json').write_text(json.dumps({'health':info,'startup_seconds':time.monotonic()-began,
|
|
'arguments':{k:str(v) if isinstance(v,Path) else v for k,v in vars(args).items()}},indent=2)+'\n',encoding='utf-8')
|
|
thread=threading.Thread(target=monitor,daemon=True); thread.start()
|
|
command=[sys.executable,'scripts/browser_multilingual.py','--url','http://127.0.0.1:8772/capture-local.html',
|
|
'--streams',str(args.streams),'--seconds',str(args.seconds),'--interval',str(args.interval),'--varied',
|
|
'--output',str(args.output/'browser.json'),'--stop-file',str(stop_file)]
|
|
if args.mixed: command.append('--mixed')
|
|
if args.rotate_modes: command.append('--rotate-modes')
|
|
result=subprocess.run(command)
|
|
finally:
|
|
finished.set()
|
|
if thread: thread.join(timeout=10)
|
|
service.stop()
|
|
(args.output/'monitor-errors.json').write_text(json.dumps(monitor_errors),encoding='utf-8')
|
|
if (args.output/'browser.json').exists():
|
|
report=summarize(json.loads((args.output/'browser.json').read_text(encoding='utf-8')),samples)
|
|
report['monitor_errors']=monitor_errors
|
|
report['passed']=report['passed'] and not monitor_errors
|
|
(args.output/'summary.json').write_text(json.dumps(report,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps({k:v for k,v in report.items() if k!='per_stream'},indent=2))
|
|
if not report['passed']: raise SystemExit('Acceptance failed; original checks and evidence retained')
|
|
if result.returncode: raise SystemExit(result.returncode)
|