diff --git a/README.md b/README.md index 68cfb9d..7ed17dd 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,29 @@ const samples = new Int16Array( // `samples.length` is frames * channels ``` +#### Packet loss concealment and FEC + +`decode()` also supports packet loss recovery. These modes require the exact +number of samples per channel in the missing packet. The value must be a +multiple of 2.5 ms at the decoder's sample rate (for example, `960` is 20 ms +at 48 kHz) and cannot exceed 120 ms. + +```js +const frameSize = 960; // 20 ms per channel at 48 kHz + +// Recover a lost packet immediately with packet loss concealment (PLC). +const concealed = encoder.decode(null, { frameSize }); + +// Or, when the next packet carries in-band FEC, recover the preceding packet. +const recovered = encoder.decode(nextPacket, { fec: true, frameSize }); + +// Decode that next packet normally after its FEC data has been consumed. +const current = encoder.decode(nextPacket); +``` + +FEC must be enabled by the sender. If the packet has no FEC data, requesting +`fec: true` falls back to PLC, as defined by libopus. + --- ### `encoder.decodeFloat(packet: Buffer): Buffer` diff --git a/src/index.ts b/src/index.ts index 97aca30..246f3a5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,27 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import nodeGypBuild from "node-gyp-build"; +export interface OpusFecDecodeOptions { + /** + * Decode in-band FEC from this packet for the immediately preceding lost + * packet. Requires `frameSize`. + */ + fec: true; + /** + * The exact number of samples per channel in the missing packet. Required + * for FEC and packet loss concealment; it must be a 2.5 ms multiple. + */ + frameSize: number; +} + +export interface OpusPlcDecodeOptions { + /** + * The exact number of samples per channel in the missing packet. It must be + * a 2.5 ms multiple. + */ + frameSize: number; +} + export interface OpusEncoder { encode(buf: Buffer): Buffer; /** @@ -10,6 +31,13 @@ export interface OpusEncoder { * @param buf Opus buffer */ decode(buf: Buffer): Buffer; + /** + * Decodes in-band FEC from an Opus packet, or uses packet loss concealment + * when `packet` is null. FEC and packet loss concealment require the exact + * missing frame size, in samples per channel. + */ + decode(packet: Buffer, options: OpusFecDecodeOptions): Buffer; + decode(packet: null, options: OpusPlcDecodeOptions): Buffer; /** * Decodes the given Opus buffer to PCM 32-bit floating-point little-endian. * @param buf Opus buffer diff --git a/src/node-opus.cc b/src/node-opus.cc index e417c09..6cf1cff 100644 --- a/src/node-opus.cc +++ b/src/node-opus.cc @@ -3,6 +3,7 @@ // Build this as part of the node‑gyp addon (binding name: opus). #include +#include #include #include "../libopus/opus/include/opus.h" @@ -11,6 +12,17 @@ // ----------------------------------------------------------------------------- static constexpr int MAX_FRAME_SIZE = 5760; // 120 ms @ 48 kHz mono static constexpr int MAX_PACKET_SIZE = 1276; // per Opus spec +static constexpr int MAX_FRAME_DURATION_MS = 120; + +static int MaxFrameSizeForRate(opus_int32 rate) +{ + return rate * MAX_FRAME_DURATION_MS / 1000; +} + +static int SamplesPer2Point5ms(opus_int32 rate) +{ + return rate / 400; +} // ----------------------------------------------------------------------------- // Utility: translate libopus error codes to strings @@ -179,9 +191,9 @@ Napi::Value OpusEncoderWrap::Encode(const Napi::CallbackInfo &info) Napi::Value OpusEncoderWrap::Decode(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); - if (info.Length() < 1 || !info[0].IsBuffer()) + if (info.Length() < 1 || (!info[0].IsBuffer() && !info[0].IsNull())) { - Napi::TypeError::New(env, "Argument must be a Buffer").ThrowAsJavaScriptException(); + Napi::TypeError::New(env, "Argument must be a Buffer or null for packet loss concealment").ThrowAsJavaScriptException(); return env.Null(); } @@ -191,8 +203,90 @@ Napi::Value OpusEncoderWrap::Decode(const Napi::CallbackInfo &info) return env.Null(); } - Napi::Buffer buf = info[0].As>(); - int dlen = opus_decode(dec_, buf.Data(), buf.Length(), outPcm_, MAX_FRAME_SIZE, 0); + const bool packetLoss = info[0].IsNull(); + bool decodeFec = false; + bool hasFrameSize = false; + int frameSize = MaxFrameSizeForRate(rate_); + + if (info.Length() > 1 && !info[1].IsUndefined()) + { + if (!info[1].IsObject() || info[1].IsNull() || info[1].IsBuffer()) + { + Napi::TypeError::New(env, "Decode options must be an object").ThrowAsJavaScriptException(); + return env.Null(); + } + + Napi::Object options = info[1].As(); + if (options.Has("fec")) + { + Napi::Value fec = options.Get("fec"); + if (!fec.IsUndefined()) + { + if (!fec.IsBoolean()) + { + Napi::TypeError::New(env, "options.fec must be a boolean").ThrowAsJavaScriptException(); + return env.Null(); + } + decodeFec = fec.As().Value(); + } + } + + if (options.Has("frameSize")) + { + Napi::Value requestedFrameSize = options.Get("frameSize"); + if (!requestedFrameSize.IsUndefined()) + { + if (!requestedFrameSize.IsNumber()) + { + Napi::TypeError::New(env, "options.frameSize must be a number").ThrowAsJavaScriptException(); + return env.Null(); + } + + double value = requestedFrameSize.As().DoubleValue(); + int maxFrameSize = MaxFrameSizeForRate(rate_); + if (!std::isfinite(value) || std::floor(value) != value || value < 1 || value > maxFrameSize) + { + Napi::RangeError::New(env, "options.frameSize must be an integer between 1 and the maximum 120 ms frame size").ThrowAsJavaScriptException(); + return env.Null(); + } + frameSize = static_cast(value); + hasFrameSize = true; + } + } + } + + if (packetLoss && decodeFec) + { + Napi::TypeError::New(env, "FEC decoding requires a packet Buffer").ThrowAsJavaScriptException(); + return env.Null(); + } + + if ((packetLoss || decodeFec) && !hasFrameSize) + { + Napi::TypeError::New(env, "options.frameSize is required when decoding FEC or packet loss concealment").ThrowAsJavaScriptException(); + return env.Null(); + } + + if (packetLoss || decodeFec) + { + int samplesPer2Point5ms = SamplesPer2Point5ms(rate_); + if (frameSize % samplesPer2Point5ms != 0) + { + Napi::RangeError::New(env, "options.frameSize must be a multiple of 2.5 ms when decoding FEC or packet loss concealment").ThrowAsJavaScriptException(); + return env.Null(); + } + } + + const unsigned char *data = nullptr; + opus_int32 length = 0; + if (!packetLoss) + { + Napi::Buffer buf = info[0].As>(); + data = buf.Data(); + length = static_cast(buf.Length()); + } + + int dlen = opus_decode(dec_, data, length, outPcm_, frameSize, decodeFec ? 1 : 0); if (dlen < 0) { Napi::Error::New(env, StrError(dlen)).ThrowAsJavaScriptException(); diff --git a/src/tests/test.js b/src/tests/test.js index fd85191..14fd7d8 100644 --- a/src/tests/test.js +++ b/src/tests/test.js @@ -8,17 +8,157 @@ const { OpusEncoder } = ? nodeGypBuild(path.resolve(import.meta.dirname, "../..")) : await import("../../dist/index.js"); +const sampleRate = 16_000; +const channels = 1; +const frameSize = 320; +const expectedPcmBytes = frameSize * channels * Int16Array.BYTES_PER_ELEMENT; + const frame = fs.readFileSync(path.join(import.meta.dirname, "frame.opus")); -const int16Decoder = new OpusEncoder(16_000, 1); -const decoded = int16Decoder.decode(frame); +function pcmEnergy(pcm) { + return new Int16Array( + pcm.buffer, + pcm.byteOffset, + pcm.byteLength / Int16Array.BYTES_PER_ELEMENT, + ).reduce((total, sample) => total + Math.abs(sample), 0); +} + +function assertStereoInterleaving(pcm, message) { + const samples = new Int16Array( + pcm.buffer, + pcm.byteOffset, + pcm.byteLength / Int16Array.BYTES_PER_ELEMENT, + ); + assert.strictEqual(samples.length % 2, 0, `${message}: expected interleaved stereo samples`); + for (let index = 0; index < samples.length; index += 2) { + assert.strictEqual( + samples[index], + samples[index + 1], + `${message}: channels differ at frame ${index / 2}`, + ); + } +} + +// The one-argument API remains the normal packet decode path. +const opus = new OpusEncoder(sampleRate, channels); +const decoded = opus.decode(frame); +assert.strictEqual(decoded.length, expectedPcmBytes, "Decoded frame length is not 640"); + +// A null packet invokes packet loss concealment for the requested missing +// duration. Decoding one valid packet first gives the decoder PLC history. +const plcDecoder = new OpusEncoder(sampleRate, channels); +plcDecoder.decode(frame); +const concealed = plcDecoder.decode(null, { frameSize }); +assert.strictEqual(concealed.length, expectedPcmBytes, "PLC frame length is not 640"); +assert(pcmEnergy(concealed) > 10_000, "PLC should produce non-silent audio after decoder history"); + +// The existing fixture is a mono packet. A stereo decoder must upmix it into +// interleaved matching channels, including when recovering a lost packet. +const stereoChannels = 2; +const stereoExpectedPcmBytes = expectedPcmBytes * stereoChannels; +const stereoPlcDecoder = new OpusEncoder(sampleRate, stereoChannels); +const stereoDecoded = stereoPlcDecoder.decode(frame); +assert.strictEqual( + stereoDecoded.length, + stereoExpectedPcmBytes, + "Stereo decoded frame length is not 1280", +); +assertStereoInterleaving(stereoDecoded, "Stereo decode"); +const stereoConcealed = stereoPlcDecoder.decode(null, { frameSize }); +assert.strictEqual( + stereoConcealed.length, + stereoExpectedPcmBytes, + "Stereo PLC frame length is not 1280", +); +assert( + pcmEnergy(stereoConcealed) > 20_000, + "Stereo PLC should produce non-silent audio after decoder history", +); +assertStereoInterleaving(stereoConcealed, "Stereo PLC"); -assert(decoded.length === 640, "Decoded int16 frame length is not 640"); +assert.throws( + () => new OpusEncoder(sampleRate, channels).decode(null), + /frameSize is required/, + "PLC must require an explicit missing frame size", +); +assert.throws( + () => new OpusEncoder(sampleRate, channels).decode(null, { frameSize: frameSize + 1 }), + /multiple of 2.5 ms/, + "PLC frame size must be a 2.5 ms multiple", +); +assert.throws( + () => new OpusEncoder(sampleRate, channels).decode(null, { frameSize: 1_921 }), + /maximum 120 ms frame size/, + "PLC frame size must not exceed 120 ms", +); -const floatDecoder = new OpusEncoder(16_000, 1); +// This is the second packet of a 48 kHz, 20 ms libopus stream encoded with +// in-band FEC. It lets the native regression test exercise FEC without using +// the addon encoder, which keeps the decoder test isolated from encoder logic. +const fecSampleRate = 48_000; +const fecFrameSize = 960; +const packetWithFec = Buffer.from( + "SMKIXWwRO6ADspF8O/7m+usoApNXAq1y8zuwT35zZOoKLJ9sEFflSpMONnQHCvAu1wl0SxsCLPt6dJKA", + "base64", +); + +const fecDecoder = new OpusEncoder(fecSampleRate, channels); +const recovered = fecDecoder.decode(packetWithFec, { fec: true, frameSize: fecFrameSize }); +assert.strictEqual( + recovered.length, + fecFrameSize * Int16Array.BYTES_PER_ELEMENT, + "FEC frame length is not 1920", +); +assert( + new Int16Array(recovered.buffer, recovered.byteOffset, recovered.byteLength / 2).some( + (sample) => sample !== 0, + ), + "FEC did not recover the previous non-silent packet", +); + +// FEC consumes the redundant data; decode the same packet normally to advance +// the decoder to the current packet. +const current = fecDecoder.decode(packetWithFec); +assert.strictEqual( + current.length, + fecFrameSize * Int16Array.BYTES_PER_ELEMENT, + "Current frame length is not 1920", +); +assert.notDeepStrictEqual( + recovered, + current, + "FEC must decode the preceding frame, not the current packet", +); + +const stereoFecDecoder = new OpusEncoder(fecSampleRate, stereoChannels); +const stereoRecovered = stereoFecDecoder.decode(packetWithFec, { + fec: true, + frameSize: fecFrameSize, +}); +assert.strictEqual( + stereoRecovered.length, + fecFrameSize * stereoChannels * Int16Array.BYTES_PER_ELEMENT, + "Stereo FEC frame length is not 3840", +); +assert(pcmEnergy(stereoRecovered) > 20_000, "Stereo FEC should recover non-silent audio"); +assertStereoInterleaving(stereoRecovered, "Stereo FEC"); + +assert.throws( + () => new OpusEncoder(fecSampleRate, channels).decode(packetWithFec, { fec: true }), + /frameSize is required/, + "FEC must require an explicit missing frame size", +); +assert.throws( + () => + new OpusEncoder(fecSampleRate, channels).decode(null, { fec: true, frameSize: fecFrameSize }), + /requires a packet Buffer/, + "FEC must reject a null packet", +); + +const floatDecoder = new OpusEncoder(sampleRate, channels); const decodedFloat = floatDecoder.decodeFloat(frame); -assert(decodedFloat.length === 1_280, "Decoded float32 frame length is not 1280"); +assert.strictEqual(decodedFloat.length, 1_280, "Decoded float32 frame length is not 1280"); const floatSamples = new Float32Array( decodedFloat.buffer, @@ -40,10 +180,14 @@ assert( "Decoded float32 samples do not match the signed 16-bit decode", ); -const stereoFloatDecoder = new OpusEncoder(16_000, 2); +const stereoFloatDecoder = new OpusEncoder(sampleRate, stereoChannels); const decodedStereoFloat = stereoFloatDecoder.decodeFloat(frame); -assert(decodedStereoFloat.length === 2_560, "Decoded stereo float32 frame length is not 2560"); +assert.strictEqual( + decodedStereoFloat.length, + 2_560, + "Decoded stereo float32 frame length is not 2560", +); const stereoFloatSamples = new Float32Array( decodedStereoFloat.buffer,