We are using a PI 4 to Windows 11 setup. The Windows camera app and the OBS app works very well.
However, we can't get it to work in OpenCV. An OpenCV expert and I have spent at least 5 hours with Gemini and ChatGPT and we still have not been able to get it to work and always get a blank screen.
We have forced DirectShow, explicitly Set Format Before Reading, hard-Flush Frames on Startup, assure One Process Owns the Camera, among many other things. Still blank screen.
Any help would be GREATLY appreciated. Here is our test file:
import subprocess
import time
import sys
import signal
import cv2
import numpy as np
def open_ffmpeg_dshow(device_name: str, width: int, height: int, fps: int):
"""
Starts FFmpeg reading from DirectShow and outputting raw BGR frames to stdout.
This bypasses OpenCV's webcam decode path (which is where VirtualHere often breaks).
"""
# Notes:
# - video_size + framerate requests the mode (driver may choose closest).
# - pixel_format "mjpeg" can help sometimes, but leave it out initially to let dshow choose.
# If you want to force MJPEG later, uncomment: "-input_format", "mjpeg"
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel", "error",
"-f", "dshow",
"-rtbufsize", "256M",
"-framerate", str(fps),
"-video_size", f"{width}x{height}",
# "-input_format", "mjpeg", # optional experiment
"-i", f"video={device_name}",
"-an",
"-pix_fmt", "bgr24",
"-f", "rawvideo",
"pipe:1",
]
# Use a new process group so we can terminate it cleanly on Windows
creationflags = subprocess.CREATE_NEW_PROCESS_GROUP if sys.platform.startswith("win") else 0
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.PIPE,
bufsize=10**8, creationflags=creationflags)
return p
def main():
# === SET THESE ===
DEVICE_NAME = "HDMI USB Camera"
WIDTH, HEIGHT, FPS = 960, 540, 30
frame_bytes = WIDTH * HEIGHT * 3
print(f"Starting FFmpeg DirectShow capture: {DEVICE_NAME} @ {WIDTH}x{HEIGHT}@{FPS}", flush=True)
p = open_ffmpeg_dshow(DEVICE_NAME, WIDTH, HEIGHT, FPS)
win = "VirtualHere via FFmpeg (bypasses OpenCV capture)"
cv2.namedWindow(win, cv2.WINDOW_NORMAL)
cv2.resizeWindow(win, max(640, WIDTH), max(360, HEIGHT))
cv2.waitKey(1)
frames = 0
t0 = time.time()
try:
while True:
# Read exactly one frame worth of bytes
raw = p.stdout.read(frame_bytes)
if raw is None or len(raw) != frame_bytes:
# If FFmpeg died, show stderr for debugging
err = b""
try:
err = p.stderr.read()
except Exception:
pass
raise RuntimeError(f"FFmpeg stream ended (got {len(raw) if raw else 0} bytes). stderr:\n{err.decode(errors='ignore')}")
frame = np.frombuffer(raw, dtype=np.uint8).reshape((HEIGHT, WIDTH, 3))
frames += 1
now = time.time()
if now - t0 >= 1.0:
print(f"FPS: {frames}", flush=True)
frames = 0
t0 = now
cv2.imshow(win, frame)
if (cv2.waitKey(1) & 0xFF) == ord("q"):
break
except KeyboardInterrupt:
print("KeyboardInterrupt. Shutting down...", flush=True)
finally:
# Clean shutdown: close window, terminate ffmpeg
try:
cv2.destroyAllWindows()
except Exception:
pass
if p and p.poll() is None:
try:
if sys.platform.startswith("win"):
# Send Ctrl+Break to the process group (more graceful than terminate)
p.send_signal(signal.CTRL_BREAK_EVENT)
time.sleep(0.5)
p.terminate()
except Exception:
pass
try:
p.wait(timeout=2.0)
except Exception:
try:
p.kill()
except Exception:
pass
print("Clean shutdown complete.", flush=True)
if __name__ == "__main__":
main()
.
You will need to stream as MJEPG because there will not be enough bandwidth in the network link to stream raw frames.
Hi,I have tried with MJEPG,…
Hi,
I have tried with MJEPG, H264, YUV2 but OpenCV is not grabbing the frames from the camera. Here is a short snippet of code that we are trying.
import cv2, timeIDX = 0 # try 0/1/2 etccap = cv2.VideoCapture(IDX, cv2.CAP_DSHOW) # <-- keyif not cap.isOpened():raise RuntimeError("Could not open camera")# Force a codec/pixel format, the below format has been tried one by one.cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"MJPG"))#
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"YUY2"))#cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc(*"H264"))time.sleep(0.2)# Force resolution + fps firstcap.set(cv2.CAP_PROP_FRAME_WIDTH, 1920)cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1080)cap.set(cv2.CAP_PROP_FPS, 30)# Warm-up: throw away initial frames (black during init is common)for _ in range(30):cap.read()while True:ok, frame = cap.read()print(frame.shape)if not ok or frame is None:continuecv2.imshow("cam", frame)if cv2.waitKey(1) & 0xFF == 27:breakcap.release()cv2.destroyAllWindows()OpenCV Version is 4.10.0 and Python version is 3.12. OpenCV is installed via PIP.
Can you help us with what we are missing?
.
on the pi try
sudo sh -c 'echo 256 > /sys/module/usbcore/parameters/usbfs_memory_mb'Black Frames With OpenCV
Even after trying this running into the issue of blank/black frames coming via OpenCV frame reading.
Similar issue has been reported on OpenCV portal (3 years ago) for this issue. https://github.com/opencv/opencv/issues/19746
Any help would be greatly appreciated.
.
I dont know unfortunatley. Try a reasolution of 160x120 mjpeg 1fps and see if that shows anything.
We made it working by using…
We made it working by using the
MSMFbackend instead ofDSHOW.Thanks for the help!
.
OK great, thanks for letting me know. That might be useful for others.