Skip to content

Auto codec detection #135

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
henkwiedig merged 5 commits into from
Jul 8, 2026
Merged

Auto codec detection #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da3696a
poc
henkwiedig Jun 26, 2026
35ed750
fix recordings on mid stream codec switch
henkwiedig Jun 26, 2026
f19783a
drop startup sniff
henkwiedig Jun 26, 2026
335442b
cleanup
henkwiedig Jun 26, 2026
8c25cf5
add icons
henkwiedig Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions src/dvr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,32 @@ void Dvr::loop() {
{
SPDLOG_DEBUG("got rpc SET_PARAMS");
if (write_ctx.f == NULL) {
break; // no open file; params are kept for the next start()
}
if (!writer_inited) {
// First valid params for this file: initialize the writer.
if (video_frm_width > 0 && video_frm_height > 0 &&
codec != VideoCodec::UNKNOWN) {
init();
}
break;
}
init();
// Writer already initialized. A codec or resolution change
// can't be applied to an open mp4 track, so roll to a fresh
// file. Unchanged params are a no-op (re-initializing the open
// mux would corrupt the recording).
if (codec != init_codec ||
video_frm_width != init_w || video_frm_height != init_h) {
spdlog::info("DVR params changed (codec/resolution); rolling to new file");
stop();
if (start() == 0) {
idr_request_record_start();
if (video_frm_width > 0 && video_frm_height > 0 &&
codec != VideoCodec::UNKNOWN) {
init();
}
}
}
break;
}
case dvr_rpc::RPC_START:
Expand All @@ -171,7 +194,7 @@ void Dvr::loop() {
}
if (start() == 0) {
idr_request_record_start();
if (video_frm_width > 0 && video_frm_height > 0) {
if (video_frm_width > 0 && video_frm_height > 0 && codec != VideoCodec::UNKNOWN) {
init();
}
}
Expand All @@ -192,7 +215,7 @@ void Dvr::loop() {
if (write_ctx.f == NULL) {
if (start() == 0) {
idr_request_record_start();
if (video_frm_width > 0 && video_frm_height > 0) {
if (video_frm_width > 0 && video_frm_height > 0 && codec != VideoCodec::UNKNOWN) {
init();
}
}
Expand Down Expand Up @@ -332,6 +355,13 @@ int Dvr::start() {
}
write_ctx.file_size = 0;
mux = MP4E_open(0 /*sequential_mode*/, mp4_fragmentation_mode, &write_ctx, write_callback);
// Fresh file: writer not yet initialized and parameter sets must be
// recached for whatever codec this file ends up carrying.
writer_inited = false;
cached_vps.clear();
cached_sps.clear();
cached_pps.clear();
params_complete = false;
if (max_file_size > 0)
spdlog::info("DVR file splitting enabled at {} MB", max_file_size / (1024*1024));
return 0;
Expand All @@ -348,6 +378,12 @@ void Dvr::init() {
spdlog::error("mp4_h26x_write_init failed");
mux = NULL;
write_ctx.f = NULL;
writer_inited = false;
} else {
writer_inited = true;
init_codec = codec;
init_w = video_frm_width;
init_h = video_frm_height;
}
_ready_to_write = 1;
if (on_start_cb) on_start_cb();
Expand All @@ -364,6 +400,7 @@ void Dvr::stop() {
write_ctx.f = NULL;
write_ctx.file_size = 0;
_ready_to_write = 0;
writer_inited = false;
}

// Check if buffer contains an IDR/IRAP slice.
Expand Down Expand Up @@ -448,6 +485,12 @@ void Dvr::split() {
video_frm_width, video_frm_height, codec == VideoCodec::H265)) {
spdlog::error("mp4_h26x_write_init failed on split file");
_ready_to_write = 0;
writer_inited = false;
} else {
writer_inited = true;
init_codec = codec;
init_w = video_frm_width;
init_h = video_frm_height;
}
}

Expand Down
7 changes: 7 additions & 0 deletions src/dvr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,13 @@ class Dvr {
uint32_t video_frm_width;
uint32_t video_frm_height;
VideoCodec codec;
// What the currently-open mp4 writer was initialized with. A mid-recording
// codec/resolution change cannot be applied to an open mp4 track, so we roll
// to a fresh file instead of re-initializing (which corrupts the mux).
VideoCodec init_codec = VideoCodec::UNKNOWN;
uint32_t init_w = 0;
uint32_t init_h = 0;
bool writer_inited = false;
int _ready_to_write = 0;
bool split_pending = false;
int split_part = 0;
Expand Down
178 changes: 175 additions & 3 deletions src/gstrtpreceiver.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ static VideoCodec detect_mp4_codec(const char* file_path) {
return result;
}

// Defined below; classifies a single RTP packet as H264/H265/UNKNOWN.
static VideoCodec classify_rtp_packet(const uint8_t* pkt, size_t len);

namespace {
static constexpr int kIdrUdpPort = 11223;
static constexpr int kIdrBurstCount = 3;
Expand All @@ -130,6 +133,22 @@ namespace {
static constexpr uint64_t kDecodeStallPktWindowMs = 500;
static constexpr uint64_t kRtpSeqResetMs = 1000;

// Number of consecutive RTP packets that must classify as the *other* codec
// before a mid-stream switch is accepted. Frequent active-codec packets reset
// the run, so a stray misclassified packet cannot trigger a rebuild.
static constexpr int kCodecSwitchConfirm = 12;

// Mid-stream codec-switch detection. g_active_codec is the codec the running
// pipeline was built for; g_codec_switch_cb performs the rebuild and is set
// by the receiver. The cb is invoked at most once until the rebuild clears
// g_codec_switch_pending.
static std::atomic<bool> g_codec_auto{false};
static std::atomic<int> g_active_codec{static_cast<int>(VideoCodec::UNKNOWN)};
static std::atomic<int> g_codec_switch_run{0};
static std::atomic<bool> g_codec_switch_pending{false};
static std::mutex g_codec_switch_mutex;
static std::function<void(VideoCodec)> g_codec_switch_cb;

static std::mutex g_idr_sock_mutex;
static int g_idr_sock = -1;
static std::atomic<bool> g_idr_sock_ready{false};
Expand Down Expand Up @@ -502,6 +521,66 @@ namespace {
g_last_rtp_seq_ms.store(now, std::memory_order_relaxed);
}

static void note_pipeline_codec(VideoCodec codec) {
g_active_codec.store(static_cast<int>(codec), std::memory_order_relaxed);
g_codec_switch_run.store(0, std::memory_order_relaxed);
g_codec_switch_pending.store(false, std::memory_order_relaxed);
}

static void set_codec_switch_callback(std::function<void(VideoCodec)> cb) {
std::lock_guard<std::mutex> lock(g_codec_switch_mutex);
g_codec_switch_cb = std::move(cb);
}

// Inspect a raw RTP packet and, if the stream has switched to the other
// codec for a sustained run, hand off a rebuild to the registered callback.
// Called from RTP-ingress threads; must not block or rebuild inline.
static void maybe_detect_codec_switch(const uint8_t* rtp, size_t len) {
if (!g_codec_auto.load(std::memory_order_relaxed)) {
return; // codec pinned by the user; never override it
}
const int active = g_active_codec.load(std::memory_order_relaxed);
if (active == static_cast<int>(VideoCodec::UNKNOWN)) {
return; // no pipeline built yet, nothing to compare against
}
if (g_codec_switch_pending.load(std::memory_order_relaxed)) {
return; // a switch is already being applied
}

const VideoCodec c = classify_rtp_packet(rtp, len);
if (c == VideoCodec::UNKNOWN) {
return; // ambiguous packet: neither confirm nor reset
}
if (static_cast<int>(c) == active) {
g_codec_switch_run.store(0, std::memory_order_relaxed);
return;
}

if (g_codec_switch_run.fetch_add(1, std::memory_order_relaxed) + 1 < kCodecSwitchConfirm) {
return;
}

// Confirmed switch; ensure we only fire once until the rebuild completes.
if (g_codec_switch_pending.exchange(true, std::memory_order_relaxed)) {
return;
}
g_codec_switch_run.store(0, std::memory_order_relaxed);
spdlog::info("[CODEC] Mid-stream switch detected: {} -> {}",
active == static_cast<int>(VideoCodec::H265) ? "H.265" : "H.264",
c == VideoCodec::H265 ? "H.265" : "H.264");

std::function<void(VideoCodec)> cb;
{
std::lock_guard<std::mutex> lock(g_codec_switch_mutex);
cb = g_codec_switch_cb;
}
if (cb) {
cb(c);
} else {
g_codec_switch_pending.store(false, std::memory_order_relaxed);
}
}

static void for_each_nal(const uint8_t* data, size_t size,
const std::function<void(const uint8_t*, size_t)>& cb) {
auto find_start = [&](size_t from, size_t& start_len) -> size_t {
Expand Down Expand Up @@ -791,6 +870,11 @@ namespace {
if (buf) {
on_incoming_stream_buffer(buf, "udpsrc");
maybe_track_rtp_sequence(buf);
GstMapInfo map;
if (gst_buffer_map(buf, &map, GST_MAP_READ)) {
maybe_detect_codec_switch(map.data, map.size);
gst_buffer_unmap(buf, &map);
}
}
}
return GST_PAD_PROBE_OK;
Expand Down Expand Up @@ -860,6 +944,43 @@ namespace {
}
}

// --- RTP codec autodetection ---------------------------------------------
// Classify a single RTP/AVP packet as H264 or H265 by inspecting the NAL
// header that follows the RTP header. Returns UNKNOWN when the packet is not a
// usable video packet or is ambiguous.
static VideoCodec classify_rtp_packet(const uint8_t* pkt, size_t len) {
if (len < RTP_HEADER_LEN + 1) return VideoCodec::UNKNOWN;
if (((pkt[0] >> 6) & 0x3) != 2) return VideoCodec::UNKNOWN; // RTP version 2
size_t off = RTP_HEADER_LEN + (pkt[0] & 0x0F) * 4; // skip CSRCs
if (pkt[0] & 0x10) { // skip extension
if (len < off + 4) return VideoCodec::UNKNOWN;
off += 4 + ((pkt[off + 2] << 8 | pkt[off + 3]) * 4);
}
if (off >= len) return VideoCodec::UNKNOWN;

const uint8_t nb = pkt[off];
if (nb & 0x80) return VideoCodec::UNKNOWN; // forbidden_zero_bit

// Fragmentation units carry the bulk of a video stream and use disjoint
// header bytes between the two codecs, so they are the most reliable signal.
if (nb == 0x1C || nb == 0x3C || nb == 0x5C || nb == 0x7C) // H264 FU-A (type 28)
return VideoCodec::H264;
if (nb == 0x62 || nb == 0x63) // H265 FU (type 49)
return VideoCodec::H265;

// Otherwise (parameter sets, SEI, single-NAL slices) prefer the codec whose
// NAL-type interpretation is valid while the other's is not, e.g. VPS(32)
// -> 0x40 is invalid as H264 type 0, H264 SPS(7) -> 0x67 is invalid as
// H265 type 51.
const uint8_t t264 = nb & 0x1F;
const uint8_t t265 = (nb >> 1) & 0x3F;
const bool h264_valid = (t264 >= 1 && t264 <= 23);
const bool h265_valid = (t265 <= 40);
if (h265_valid && !h264_valid) return VideoCodec::H265;
if (h264_valid && !h265_valid) return VideoCodec::H264;
return VideoCodec::UNKNOWN;
}

static void initGstreamerOrThrow() {
GError* error = nullptr;
if (!gst_init_check(nullptr, nullptr, &error)) {
Expand All @@ -872,13 +993,15 @@ GstRtpReceiver::GstRtpReceiver(int udp_port, const VideoCodec& codec)
{
m_port=udp_port;
m_video_codec=codec;
m_auto_codec = (codec == VideoCodec::UNKNOWN);
initGstreamerOrThrow();

}

GstRtpReceiver::GstRtpReceiver(const char *s, const VideoCodec& codec) {
unix_socket = strdup(s);
m_video_codec = codec;
m_auto_codec = (codec == VideoCodec::UNKNOWN);
initGstreamerOrThrow();

spdlog::debug("Creating receiver socket on {}", unix_socket);
Expand Down Expand Up @@ -909,6 +1032,11 @@ GstRtpReceiver::GstRtpReceiver(const char *s, const VideoCodec& codec) {
}

GstRtpReceiver::~GstRtpReceiver(){
// Drop the detection callback so the ingress threads can never call back
// into a destroyed receiver.
set_codec_switch_callback(nullptr);
g_codec_auto.store(false, std::memory_order_relaxed);
note_pipeline_codec(VideoCodec::UNKNOWN);
if (sock >= 0) {
close(sock);
}
Expand Down Expand Up @@ -1022,8 +1150,11 @@ static void loop_read_socket(bool& keep_looping, int sock_fd, GstAppSrc* appsrc)

// Read data directly into buffer
ssize_t n = recv(sock_fd, map.data, map.size, 0);
if (n > 0) {
maybe_detect_codec_switch(map.data, static_cast<size_t>(n));
}
gst_buffer_unmap(buffer, &map);

if (n <= RTP_HEADER_LEN) {
spdlog::warn("Invalid RTP packet size: {}", n);
gst_buffer_unref(buffer);
Expand Down Expand Up @@ -1145,7 +1276,17 @@ VideoCodec GstRtpReceiver::switch_to_file_playback(const char * file_path) {

void GstRtpReceiver::switch_to_stream() {
stop_receiving();


// Auto mode: build for H.265 up front and let mid-stream detection flip to
// H.264 from the RTP ingress if the stream turns out to be H.264. The
// ingress classifier sees raw RTP before depay/parse, so it works even
// though the initial pipeline guesses wrong, and this avoids blocking
// startup to sniff (which would stall when no stream is live yet).
if (m_video_codec == VideoCodec::UNKNOWN) {
m_video_codec = VideoCodec::H265;
spdlog::info("[CODEC] Auto mode: defaulting to H.265; mid-stream detection will correct if needed");
}

const auto pipeline = construct_gstreamer_pipeline();
GError* error = nullptr;
m_gst_pipeline = gst_parse_launch(pipeline.c_str(), &error);
Expand Down Expand Up @@ -1220,9 +1361,40 @@ void GstRtpReceiver::switch_to_stream() {
assert(m_app_sink_element);

gst_element_set_state(m_gst_pipeline, GST_STATE_PLAYING);

m_pull_samples_run = true;
m_pull_samples_thread = std::make_unique<std::thread>(&GstRtpReceiver::loop_pull_samples, this);

// Arm mid-stream codec-switch detection for the codec we just built for,
// but only in auto mode (a pinned codec is never overridden).
g_codec_auto.store(m_auto_codec, std::memory_order_relaxed);
set_codec_switch_callback([this](VideoCodec new_codec) {
request_codec_switch(new_codec);
});
note_pipeline_codec(m_video_codec);
}

void GstRtpReceiver::request_codec_switch(VideoCodec new_codec) {
// Rebuild on a detached thread: switch_to_stream() tears down the pipeline
// (set_state NULL) and joins the pull/socket threads, neither of which is
// safe to do from a GStreamer streaming thread or the socket reader itself.
std::thread([this, new_codec]() {
m_video_codec = new_codec; // non-UNKNOWN: switch_to_stream keeps it as-is
switch_to_stream(); // also clears the pending flag
std::function<void(VideoCodec)> cb;
{
std::lock_guard<std::mutex> lock(m_codec_changed_mutex);
cb = m_on_codec_changed;
}
if (cb) {
cb(new_codec); // let the host realign its decoder
}
}).detach();
}

void GstRtpReceiver::set_codec_changed_callback(std::function<void(VideoCodec)> cb) {
std::lock_guard<std::mutex> lock(m_codec_changed_mutex);
m_on_codec_changed = std::move(cb);
}

void GstRtpReceiver::set_playback_rate(double rate) {
Expand Down
Loading