Logs for troubleshooting

bash <<'EOF'
set -euo pipefail
sudo apt-get update
sudo apt-get install -y python3.12 python3.12-venv python3-pip curl unzip tar gzip ipmitool
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=/tmp/nscale_manual_logs_$(hostname -s)_$(date -u +%Y%m%dT%H%M%SZ)
mkdir -p "$LOGDIR"
printf '%s\n' "$LOGDIR" > /tmp/nscale_manual_logdir.path
echo "Log directory: $LOGDIR"
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
NVDEBUG_TAG=nvdebug-v2.1.0-release
curl -L -o open-nvdebug.zip "https://github.com/NVIDIA/open-nvdebug/archive/refs/tags/${NVDEBUG_TAG}.zip"
unzip open-nvdebug.zip
cd "open-nvdebug-${NVDEBUG_TAG}"

python3.12 -m venv .venv
. .venv/bin/activate
python -m pip install --upgrade pip setuptools wheel
python -m pip install -r requirements.txt
python -m src.tool.main --version > "$LOGDIR/nvdebug-version.txt" 2>&1
printf '%s\n' "$PWD" > /tmp/nscale_manual_nvdebug_dir.path
echo "NVDebug prepared under: $PWD"
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
NVDEBUG_DIR=$(cat /tmp/nscale_manual_nvdebug_dir.path)
DUT_CONFIG="$LOGDIR/nvdebug-local-host-dut_config.json"
: > "$DUT_CONFIG"
chmod 600 "$DUT_CONFIG"

"$NVDEBUG_DIR/.venv/bin/python" - "$DUT_CONFIG" <<'PYCONFIG'
import getpass
import json
import sys
from pathlib import Path

bmc_ip = input("BMC IP or hostname: ").strip()
bmc_user = input("BMC username: ").strip()
bmc_pass = getpass.getpass("BMC password: ")
baseboard = input("Baseboard override (optional, press Enter for auto-detect): ").strip()

config = {
    "dut-1": {
        "local": True,
        "baseboard": baseboard,
        "BMC_IP": bmc_ip,
        "BMC_USERNAME": bmc_user,
        "BMC_PASSWORD": bmc_pass,
    }
}

Path(sys.argv[1]).write_text(json.dumps(config, indent=2))
PYCONFIG

cd "$NVDEBUG_DIR"
trap 'rm -f "$DUT_CONFIG"' EXIT
sudo "$PWD/.venv/bin/python" -m src.tool.main collect --dut-config "$DUT_CONFIG" -o "$LOGDIR/nvdebug-host-bmc" > "$LOGDIR/nvdebug-host-bmc.stdout.txt" 2>&1
echo "NVDebug generated files: $LOGDIR/nvdebug-host-bmc"
echo "NVDebug console log: $LOGDIR/nvdebug-host-bmc.stdout.txt"
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
(cd "$LOGDIR" && sudo nvidia-bug-report.sh) > "$LOGDIR/nvidia-bug-report.stdout.txt" 2>&1
echo "NVIDIA bug report output saved under: $LOGDIR"
EOF

bash <<'EOF'
set -u
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
nvidia-smi -q > nvidia-smi-q.txt 2>&1
nvidia-smi topo -m > nvidia-smi-topo-m.txt 2>&1
nvidia-smi nvlink -s > nvidia-smi-nvlink-s.txt 2>&1
nvidia-smi nvlink --errorcounters > nvidia-smi-nvlink-errorcounters.txt 2>&1
nvidia-smi --query-gpu=index,name,uuid,serial,pci.bus_id,temperature.gpu,memory.total,memory.used,memory.free,power.draw --format=csv > nvidia-smi-query-gpu-health.csv 2>&1
nvidia-smi --query-gpu=index,serial,uuid,pci.bus_id,fabric.state,fabric.status --format=csv > nvidia-smi-query-fabric-status.csv 2>&1
sudo dmesg -T | grep -Ei 'Xid|SXid|NVRM|fallen off|PCIe|AER|nvidia|nvswitch|nvlink|fabric' > dmesg-gpu-fabric-filtered.txt 2>&1
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
sudo dcgmi diag -r 4 > dcgmi-diag-r4.txt 2>&1
EOF

bash <<'EOF'
set -u
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
nvidia-smi -q -d ECC > nvidia-smi-q-d-ecc.txt 2>&1
nvidia-smi -q -d MEMORY > nvidia-smi-q-d-memory.txt 2>&1
nvidia-smi -q -d ROW_REMAPPER > nvidia-smi-q-d-row-remapper.txt 2>&1
nvidia-smi -q -d PAGE_RETIREMENT > nvidia-smi-q-d-page-retirement.txt 2>&1
sudo dmesg -T | grep -Ei 'ECC|row|remap|retire|SRAM|DRAM|memory error|Xid|NVRM|MCE|EDAC|RAS' > dmesg-memory-ecc-filtered.txt 2>&1
EOF

bash <<'EOF'
set -u
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
sudo journalctl --list-boots --no-pager > journalctl-list-boots.txt 2>&1
last -x reboot shutdown -F > last-reboot-shutdown.txt 2>&1
who -b > who-b.txt 2>&1
uptime -s > uptime-s.txt 2>&1
EOF

bash <<'EOF'
set -u
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
df -hT > df-hT.txt 2>&1
lsblk -a -o NAME,KNAME,PATH,SIZE,TYPE,FSTYPE,MOUNTPOINTS,MODEL,SERIAL,WWN,VENDOR,STATE,TRAN > lsblk.txt 2>&1
cat /proc/mdstat > proc-mdstat.txt 2>&1
nvme list > nvme-list.txt 2>&1
for dev in /dev/nvme*n1; do [ -e "$dev" ] || continue; sudo nvme smart-log "$dev"; done > nvme-smart-log.txt 2>&1
for dev in /dev/nvme*n1; do [ -e "$dev" ] || continue; sudo nvme error-log "$dev"; done > nvme-error-log.txt 2>&1
sudo dmesg -T | grep -Ei 'I/O error|blk_update_request|buffer I/O|medium error|critical warning|nvme|md/|raid|scsi|sas|sata|smart|xfs|ext4|btrfs|reset|timeout' > dmesg-storage-filtered.txt 2>&1
EOF

bash <<'EOF'
set -u
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
cd "$LOGDIR"
free -h > free-h.txt 2>&1
cat /proc/meminfo > proc-meminfo.txt 2>&1
lscpu > lscpu.txt 2>&1
sudo dmidecode -t memory > dmidecode-memory.txt 2>&1
sensors > sensors.txt 2>&1
sudo dmesg -T | grep -Ei 'MCE|EDAC|RAS|ECC|memory error|hardware error|machine check|out of memory|oom|thermal|overheat|temperature|throttl' > dmesg-memory-ras-thermal-filtered.txt 2>&1
EOF

bash <<'EOF'
set -euo pipefail
LOGDIR=$(cat /tmp/nscale_manual_logdir.path)
ARCHIVE=/tmp/$(basename "$LOGDIR").tar.gz
sudo tar -czf "$ARCHIVE" -C /tmp "$(basename "$LOGDIR")"
sudo chown "$(id -u):$(id -g)" "$ARCHIVE" 2>/dev/null || true
echo "$ARCHIVE"
EOF