From 5d38b0a231cfd77ae598291451c23080fa53cd13 Mon Sep 17 00:00:00 2001 From: Juliusz Chroboczek Date: Fri, 25 Dec 2020 17:33:44 +0100 Subject: [PATCH] Allow various codecs. It is now possible to specify codecs other than VP8 and Opus. This turns out not to be very useful, since VP8 is the only codec supported by all browsers (in violation of the WebRTC spec, which mandates support for H.264), and there is no good reason to use anything other than Opus for audio. --- README | 5 ++ conn/conn.go | 1 + group/group.go | 181 +++++++++++++++++++++++++++++++++---------- rtpconn/rtp_test.go | 81 +++++++++++++++++++ rtpconn/rtpconn.go | 14 +++- rtpconn/rtpreader.go | 89 +++++++++++++++++++++ 6 files changed, 331 insertions(+), 40 deletions(-) diff --git a/README b/README index 97599ca..ca67a01 100644 --- a/README +++ b/README @@ -139,6 +139,11 @@ fields, all of which are optional. are automatically created when accessed. - `redirect`: if set, then attempts to join the group will be redirected to the given URL; most other fields are ignored in this case. + - `codecs`: this is a list of codecs allowed in this group. The default + is `["vp8", "opus"]`. Other possible values include `"vp9"` + (incompatible with Mac OS), `"h264"` (incompatible with some versions + of Firefox and Chromium), `"g722"`, `"pcmu"` and `"pcma"`. Recording + to disk is only supported for `"vp8"` and `"opus"`. A user definition is a dictionary with the following fields: diff --git a/conn/conn.go b/conn/conn.go index a0d0200..ae309dc 100644 --- a/conn/conn.go +++ b/conn/conn.go @@ -17,6 +17,7 @@ type Up interface { DelLocal(Down) bool Id() string Label() string + Codecs() []webrtc.RTPCodecCapability } // Type UpTrack represents a track in the client to server direction. diff --git a/group/group.go b/group/group.go index 42c3881..3926328 100644 --- a/group/group.go +++ b/group/group.go @@ -92,6 +92,7 @@ const ( type Group struct { name string + api *webrtc.API mu sync.Mutex description *description @@ -146,11 +147,131 @@ func (g *Group) AllowRecording() bool { var groups struct { mu sync.Mutex groups map[string]*Group - api *webrtc.API } func (g *Group) API() *webrtc.API { - return groups.api + return g.api +} + +func codecFromName(name string) (webrtc.RTPCodecCapability, error) { + switch name { + case "vp8": + return webrtc.RTPCodecCapability{ + "video/VP8", 90000, 0, + "", + nil, + }, nil + case "vp9": + return webrtc.RTPCodecCapability{ + "video/VP9", 90000, 0, + "profile-id=2", + nil, + }, nil + case "h264": + return webrtc.RTPCodecCapability{ + "video/H264", 90000, 0, + "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f", + nil, + }, nil + case "opus": + return webrtc.RTPCodecCapability{ + "audio/opus", 48000, 2, + "minptime=10;useinbandfec=1", + nil, + }, nil + case "g722": + return webrtc.RTPCodecCapability{ + "audio/G722", 8000, 1, + "", + nil, + }, nil + case "pcmu": + return webrtc.RTPCodecCapability{ + "audio/PCMU", 8000, 1, + "", + nil, + }, nil + case "pcma": + return webrtc.RTPCodecCapability{ + "audio/PCMA", 8000, 1, + "", + nil, + }, nil + default: + return webrtc.RTPCodecCapability{}, errors.New("unknown codec") + } +} + +func payloadType(codec webrtc.RTPCodecCapability) (webrtc.PayloadType, error) { + switch strings.ToLower(codec.MimeType) { + case "video/vp8": + return 96, nil + case "video/vp9": + return 98, nil + case "video/h264": + return 102, nil + case "audio/opus": + return 111, nil + case "audio/g722": + return 9, nil + case "audio/pcmu": + return 0, nil + case "audio/pcma": + return 8, nil + default: + return 0, errors.New("unknown codec") + } +} + +func APIFromCodecs(codecs []webrtc.RTPCodecCapability) *webrtc.API { + s := webrtc.SettingEngine{} + s.SetSRTPReplayProtectionWindow(512) + if !UseMDNS { + s.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + } + m := webrtc.MediaEngine{} + + for _, codec := range codecs { + var tpe webrtc.RTPCodecType + var fb []webrtc.RTCPFeedback + if strings.HasPrefix(strings.ToLower(codec.MimeType), "video/") { + tpe = webrtc.RTPCodecTypeVideo + fb = []webrtc.RTCPFeedback{ + {"goog-remb", ""}, + {"nack", ""}, + {"nack", "pli"}, + {"ccm", "fir"}, + } + } else if strings.HasPrefix(strings.ToLower(codec.MimeType), "audio/") { + tpe = webrtc.RTPCodecTypeAudio + fb = []webrtc.RTCPFeedback{} + } else { + continue + } + + ptpe, err := payloadType(codec) + if err != nil { + log.Printf("%v", err) + continue + } + m.RegisterCodec( + webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: codec.MimeType, + ClockRate: codec.ClockRate, + Channels: codec.Channels, + SDPFmtpLine: codec.SDPFmtpLine, + RTCPFeedback: fb, + }, + PayloadType: ptpe, + }, + tpe, + ) + } + return webrtc.NewAPI( + webrtc.WithSettingEngine(s), + webrtc.WithMediaEngine(&m), + ) } func Add(name string, desc *description) (*Group, error) { @@ -163,43 +284,6 @@ func Add(name string, desc *description) (*Group, error) { if groups.groups == nil { groups.groups = make(map[string]*Group) - s := webrtc.SettingEngine{} - s.SetSRTPReplayProtectionWindow(512) - if !UseMDNS { - s.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) - } - m := webrtc.MediaEngine{} - m.RegisterCodec( - webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - "video/VP8", 90000, 0, - "", - []webrtc.RTCPFeedback{ - {"goog-remb", ""}, - {"nack", ""}, - {"nack", "pli"}, - {"ccm", "fir"}, - }, - }, - PayloadType: 96, - }, - webrtc.RTPCodecTypeVideo, - ) - m.RegisterCodec( - webrtc.RTPCodecParameters{ - RTPCodecCapability: webrtc.RTPCodecCapability{ - "audio/opus", 48000, 2, - "minptime=10;useinbandfec=1", - nil, - }, - PayloadType: 111, - }, - webrtc.RTPCodecTypeAudio, - ) - groups.api = webrtc.NewAPI( - webrtc.WithSettingEngine(s), - webrtc.WithMediaEngine(&m), - ) } var err error @@ -212,11 +296,29 @@ func Add(name string, desc *description) (*Group, error) { return nil, err } } + + names := desc.Codecs + if len(names) == 0 { + names = []string{"vp8", "opus"} + } + codecs := make([]webrtc.RTPCodecCapability, 0, len(names)) + for _, n := range names { + codec, err := codecFromName(n) + if err != nil { + log.Printf("Codec %v: %v", n, err) + continue + } + codecs = append(codecs, codec) + } + + api := APIFromCodecs(codecs) + g = &Group{ name: name, description: desc, clients: make(map[string]Client), timestamp: time.Now(), + api: api, } groups.groups[name] = g return g, nil @@ -592,6 +694,7 @@ type description struct { Op []ClientCredentials `json:"op,omitempty"` Presenter []ClientCredentials `json:"presenter,omitempty"` Other []ClientCredentials `json:"other,omitempty"` + Codecs []string `json:"codecs,omitempty"` } const DefaultMaxHistoryAge = 4 * time.Hour diff --git a/rtpconn/rtp_test.go b/rtpconn/rtp_test.go index 3cf8099..2ae3755 100644 --- a/rtpconn/rtp_test.go +++ b/rtpconn/rtp_test.go @@ -46,3 +46,84 @@ func TestVP8Keyframe(t *testing.T) { } } } + +func TestVP9Keyframe(t *testing.T) { + ps := [][]byte{ + { + 0x80, 0xe2, 0x6c, 0xb9, 0xcd, 0xa2, 0x77, 0x5c, + 0xea, 0xf0, 0x14, 0xe9, 0x8f, 0xbd, 0x90, 0x18, + 0x0, 0x10, 0x0, 0x10, 0x1, 0x4, 0x1, 0x82, 0x49, + 0x83, 0x42, 0x0, 0x0, 0xf0, 0x0, 0xf4, 0x2, 0x38, + 0x24, 0x1c, 0x18, 0x10, 0x0, 0x0, 0x20, 0x40, 0x0, + 0x22, 0x9b, 0xff, 0xff, 0xd7, 0xe6, 0xc0, 0xa, + 0xf2, 0x32, 0xd4, 0xdd, 0xa3, 0x69, 0xc6, 0xca, + 0xd1, 0x50, 0xeb, 0x1c, 0x1, 0x50, 0x91, 0xf6, + 0x64, 0xc7, 0x35, 0xe9, 0x0, 0xfe, 0x76, 0xb2, + 0xb, 0x4d, 0xd7, 0x35, 0x23, 0xf3, 0x9f, 0x7f, + 0x86, 0x37, 0xb9, 0x65, 0x3a, 0xf9, 0x66, 0xa0, + 0x6a, 0xb2, 0x9b, 0xb3, 0x36, 0x5b, 0x47, 0xf2, + 0x26, 0x5c, 0xe2, 0x23, 0x4f, 0xff, 0xff, 0xff, + 0xfe, 0xc3, 0x49, 0x6b, 0x14, 0x58, 0x4d, 0xdc, + 0xd8, 0xf5, 0x76, 0x81, 0x2e, 0xb3, 0x7f, 0xff, + 0xfe, 0x18, 0xc8, 0xf8, 0x1b, 0xf6, 0xee, 0xc3, + 0xc, 0x6f, 0x23, 0x34, 0x80, + }, + { + 0x80, 0xe2, 0x4a, 0xb5, 0x1a, 0x33, 0x3f, 0x7b, + 0x9c, 0xda, 0x7b, 0xd0, 0x8d, 0xec, 0x14, 0x86, + 0x0, 0x40, 0x92, 0x88, 0x2c, 0x50, 0x83, 0x30, + 0x10, 0x1c, 0x6, 0x3, 0x0, 0x82, 0x99, 0x15, 0xc8, + 0x0, 0x0, 0x0, 0x0, 0x18, 0x70, 0x0, 0x0, 0x4c, + 0x4, 0xa0, + }, + } + + var packet rtp.Packet + + for i, p := range ps { + err := packet.Unmarshal(p) + if err != nil { + t.Errorf("Unmarshal(p%v): %v", i, err) + } + + kf, kfKnown := isKeyframe("video/vp9", &packet) + if kf != (i == 0) || !kfKnown { + t.Errorf("isKeyframe(p%v): %v %v", i, kf, kfKnown) + } + } +} + +func TestH264Keyframe(t *testing.T) { + ps := [][]byte{ + { + 0x80, 0xe6, 0xf, 0xae, 0xfa, 0x86, 0x3b, 0x49, + 0x59, 0xbd, 0x79, 0xe7, 0x78, 0x0, 0xc, 0x67, + 0x42, 0xc0, 0xc, 0x8c, 0x8d, 0x4e, 0x40, 0x3c, + 0x22, 0x11, 0xa8, 0x0, 0x4, 0x68, 0xce, 0x3c, + 0x80, 0x0, 0x1a, 0x65, 0xb8, 0x0, 0x4, 0x0, 0x0, + 0x9, 0xe3, 0x31, 0x40, 0x0, 0x46, 0x76, 0x38, 0x0, + 0x8, 0x2, 0x47, 0x0, 0x2, 0x7f, 0x3f, 0x77, 0x6f, + 0x67, 0x80, + }, + { + + 0x80, 0xe6, 0xf, 0xaf, 0xfa, 0x86, 0x46, 0x89, + 0x59, 0xbd, 0x79, 0xe7, 0x61, 0xe0, 0x0, 0x40, + 0x0, 0xbe, 0x40, 0x9e, 0xa0, + }, + } + + var packet rtp.Packet + + for i, p := range ps { + err := packet.Unmarshal(p) + if err != nil { + t.Errorf("Unmarshal(p%v): %v", i, err) + } + + kf, kfKnown := isKeyframe("video/h264", &packet) + if kf != (i == 0) || !kfKnown { + t.Errorf("isKeyframe(p%v): %v %v", i, kf, kfKnown) + } + } +} diff --git a/rtpconn/rtpconn.go b/rtpconn/rtpconn.go index f916694..e1d165d 100644 --- a/rtpconn/rtpconn.go +++ b/rtpconn/rtpconn.go @@ -108,7 +108,8 @@ type rtpDownConnection struct { } func newDownConn(c group.Client, id string, remote conn.Up) (*rtpDownConnection, error) { - pc, err := c.Group().API().NewPeerConnection(group.IceConfiguration()) + api := group.APIFromCodecs(remote.Codecs()) + pc, err := api.NewPeerConnection(group.IceConfiguration()) if err != nil { return nil, err } @@ -300,6 +301,17 @@ func (up *rtpUpConnection) Label() string { return up.label } +func (up *rtpUpConnection) Codecs() []webrtc.RTPCodecCapability { + up.mu.Lock() + defer up.mu.Unlock() + + codecs := make([]webrtc.RTPCodecCapability, len(up.tracks)) + for i := range up.tracks { + codecs[i] = up.tracks[i].Codec() + } + return codecs +} + func (up *rtpUpConnection) AddLocal(local conn.Down) error { up.mu.Lock() defer up.mu.Unlock() diff --git a/rtpconn/rtpreader.go b/rtpconn/rtpreader.go index 47645ca..e767b43 100644 --- a/rtpconn/rtpreader.go +++ b/rtpconn/rtpreader.go @@ -30,6 +30,95 @@ func isKeyframe(codec string, packet *rtp.Packet) (bool, bool) { return true, true } return false, true + case "video/vp9": + var vp9 codecs.VP9Packet + _, err := vp9.Unmarshal(packet.Payload) + if err != nil || len(vp9.Payload) < 1 { + return false, false + } + if !vp9.B { + return false, true + } + + if (vp9.Payload[0] & 0xc0) != 0x80 { + return false, false + } + + profile := (vp9.Payload[0] >> 4) & 0x3 + if profile != 3 { + if (vp9.Payload[0] & 0x8) != 0 { + return false, true + } + return (vp9.Payload[0] & 0x4) == 0, true + } else { + if (vp9.Payload[0] & 0x4) != 0 { + return false, true + } + return (vp9.Payload[0] & 0x2) == 0, true + } + case "video/h264": + if len(packet.Payload) < 1 { + return false, false + } + nalu := packet.Payload[0] & 0x1F + if nalu == 0 { + // reserved + return false, false + } else if nalu <= 23 { + // simple NALU + return nalu == 5, true + } else if nalu == 24 || nalu == 25 || nalu == 26 || nalu == 27 { + // STAP-A, STAP-B, MTAP16 or MTAP24 + i := 1 + if nalu == 25 || nalu == 26 || nalu == 27 { + // skip DON + i += 2 + } + for i < len(packet.Payload) { + if i+2 > len(packet.Payload) { + return false, false + } + length := uint16(packet.Payload[i])<<8 | + uint16(packet.Payload[i+1]) + i += 2 + if i+int(length) > len(packet.Payload) { + return false, false + } + offset := 0 + if nalu == 26 { + offset = 3 + } else if nalu == 27 { + offset = 4 + } + if offset >= int(length) { + return false, false + } + n := packet.Payload[i + offset] & 0x1F + if n == 5 { + return true, true + } else if n >= 24 { + // is this legal? + return false, false + } + i += int(length) + } + if i == len(packet.Payload) { + return false, true + } + return false, false + } else if nalu == 28 || nalu == 29 { + // FU-A or FU-B + if len(packet.Payload) < 2 { + return false, false + } + if (packet.Payload[1] & 0x80) == 0 { + // not a starting fragment + return false, true + } + return (packet.Payload[1]&0x1F == 5), true + } + return false, false + default: return false, false }