From 43ec0c062467c05a66eac7a3fb992bc093315c89 Mon Sep 17 00:00:00 2001
From: trilene <trilene@runbox.com>
Date: Sun, 26 Jul 2020 10:59:50 -0400
Subject: [PATCH] Handle ICE failure
---
src/ActiveCallBar.cpp | 7 +++
src/CallManager.cpp | 96 +++++++++++++++++++++++++++--------------
src/CallManager.h | 8 ++--
src/ChatPage.cpp | 9 ----
src/TextInputWidget.cpp | 3 +-
src/WebRTCSession.cpp | 65 +++++++++++++++++++---------
src/WebRTCSession.h | 4 +-
7 files changed, 125 insertions(+), 67 deletions(-)
diff --git a/src/ActiveCallBar.cpp b/src/ActiveCallBar.cpp
index 564842daf..e55b2e86e 100644
--- a/src/ActiveCallBar.cpp
+++ b/src/ActiveCallBar.cpp
@@ -123,25 +123,32 @@ ActiveCallBar::update(WebRTCSession::State state)
{
switch (state) {
case WebRTCSession::State::INITIATING:
+ show();
stateLabel_->setText("Initiating call...");
break;
case WebRTCSession::State::INITIATED:
+ show();
stateLabel_->setText("Call initiated...");
break;
case WebRTCSession::State::OFFERSENT:
+ show();
stateLabel_->setText("Calling...");
break;
case WebRTCSession::State::CONNECTING:
+ show();
stateLabel_->setText("Connecting...");
break;
case WebRTCSession::State::CONNECTED:
+ show();
callStartTime_ = QDateTime::currentSecsSinceEpoch();
timer_->start(1000);
stateLabel_->setText("Voice call:");
durationLabel_->setText("00:00");
durationLabel_->show();
break;
+ case WebRTCSession::State::ICEFAILED:
case WebRTCSession::State::DISCONNECTED:
+ hide();
timer_->stop();
callPartyLabel_->setText(QString());
stateLabel_->setText(QString());
diff --git a/src/CallManager.cpp b/src/CallManager.cpp
index 3caa812de..b57ef1bbc 100644
--- a/src/CallManager.cpp
+++ b/src/CallManager.cpp
@@ -11,9 +11,10 @@
#include "MatrixClient.h"
#include "UserSettingsPage.h"
#include "WebRTCSession.h"
-
#include "dialogs/AcceptCall.h"
+#include "mtx/responses/turn_server.hpp"
+
Q_DECLARE_METATYPE(std::vector<mtx::events::msg::CallCandidates::Candidate>)
Q_DECLARE_METATYPE(mtx::events::msg::CallCandidates::Candidate)
Q_DECLARE_METATYPE(mtx::responses::TurnServer)
@@ -24,6 +25,11 @@ using namespace mtx::events::msg;
// https://github.com/vector-im/riot-web/issues/10173
#define STUN_SERVER "stun://turn.matrix.org:3478"
+namespace {
+std::vector<std::string>
+getTurnURIs(const mtx::responses::TurnServer &turnServer);
+}
+
CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
: QObject(),
session_(WebRTCSession::instance()),
@@ -80,15 +86,23 @@ CallManager::CallManager(QSharedPointer<UserSettings> userSettings)
// Request new credentials close to expiry
// See https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00
- turnServer_ = res;
+ turnURIs_ = getTurnURIs(res);
turnServerTimer_.setInterval(res.ttl * 1000 * 0.9);
});
connect(&session_, &WebRTCSession::stateChanged, this,
[this](WebRTCSession::State state) {
- if (state == WebRTCSession::State::DISCONNECTED)
+ if (state == WebRTCSession::State::DISCONNECTED) {
playRingtone("qrc:/media/media/callend.ogg", false);
- });
+ }
+ else if (state == WebRTCSession::State::ICEFAILED) {
+ QString error("Call connection failed.");
+ if (turnURIs_.empty())
+ error += " Your homeserver has no configured TURN server.";
+ emit ChatPage::instance()->showNotification(error);
+ hangUp(CallHangUp::Reason::ICEFailed);
+ }
+ });
connect(&player_, &QMediaPlayer::mediaStatusChanged, this,
[this](QMediaPlayer::MediaStatus status) {
@@ -116,8 +130,8 @@ CallManager::sendInvite(const QString &roomid)
}
roomid_ = roomid;
- setTurnServers();
session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
+ session_.setTurnServers(turnURIs_);
generateCallID();
nhlog::ui()->debug("WebRTC: call id: {} - creating invite", callid_);
@@ -132,11 +146,26 @@ CallManager::sendInvite(const QString &roomid)
}
}
+namespace {
+std::string callHangUpReasonString(CallHangUp::Reason reason)
+{
+ switch (reason) {
+ case CallHangUp::Reason::ICEFailed:
+ return "ICE failed";
+ case CallHangUp::Reason::InviteTimeOut:
+ return "Invite time out";
+ default:
+ return "User";
+ }
+}
+}
+
void
CallManager::hangUp(CallHangUp::Reason reason)
{
if (!callid_.empty()) {
- nhlog::ui()->debug("WebRTC: call id: {} - hanging up", callid_);
+ nhlog::ui()->debug("WebRTC: call id: {} - hanging up ({})", callid_,
+ callHangUpReasonString(reason));
emit newMessage(roomid_, CallHangUp{callid_, 0, reason});
endCall();
}
@@ -221,8 +250,8 @@ CallManager::answerInvite(const CallInvite &invite)
return;
}
- setTurnServers();
session_.setStunServer(settings_->useStunServer() ? STUN_SERVER : "");
+ session_.setTurnServers(turnURIs_);
if (!session_.acceptOffer(invite.sdp)) {
emit ChatPage::instance()->showNotification("Problem setting up call.");
@@ -279,8 +308,9 @@ CallManager::handleEvent(const RoomEvent<CallAnswer> &callAnswerEvent)
void
CallManager::handleEvent(const RoomEvent<CallHangUp> &callHangUpEvent)
{
- nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp from {}",
- callHangUpEvent.content.call_id, callHangUpEvent.sender);
+ nhlog::ui()->debug("WebRTC: call id: {} - incoming CallHangUp ({}) from {}",
+ callHangUpEvent.content.call_id, callHangUpReasonString(callHangUpEvent.content.reason),
+ callHangUpEvent.sender);
if (callid_ == callHangUpEvent.content.call_id) {
MainWindow::instance()->hideOverlay();
@@ -320,12 +350,30 @@ CallManager::retrieveTurnServer()
}
void
-CallManager::setTurnServers()
+CallManager::playRingtone(const QString &ringtone, bool repeat)
+{
+ static QMediaPlaylist playlist;
+ playlist.clear();
+ playlist.setPlaybackMode(repeat ? QMediaPlaylist::CurrentItemInLoop : QMediaPlaylist::CurrentItemOnce);
+ playlist.addMedia(QUrl(ringtone));
+ player_.setVolume(100);
+ player_.setPlaylist(&playlist);
+}
+
+void
+CallManager::stopRingtone()
+{
+ player_.setPlaylist(nullptr);
+}
+
+namespace {
+std::vector<std::string>
+getTurnURIs(const mtx::responses::TurnServer &turnServer)
{
// gstreamer expects: turn(s)://username:password@host:port?transport=udp(tcp)
// where username and password are percent-encoded
- std::vector<std::string> uris;
- for (const auto &uri : turnServer_.uris) {
+ std::vector<std::string> ret;
+ for (const auto &uri : turnServer.uris) {
if (auto c = uri.find(':'); c == std::string::npos) {
nhlog::ui()->error("Invalid TURN server uri: {}", uri);
continue;
@@ -338,29 +386,13 @@ CallManager::setTurnServers()
}
QString encodedUri = QString::fromStdString(scheme) + "://" +
- QUrl::toPercentEncoding(QString::fromStdString(turnServer_.username)) + ":" +
- QUrl::toPercentEncoding(QString::fromStdString(turnServer_.password)) + "@" +
+ QUrl::toPercentEncoding(QString::fromStdString(turnServer.username)) + ":" +
+ QUrl::toPercentEncoding(QString::fromStdString(turnServer.password)) + "@" +
QString::fromStdString(std::string(uri, ++c));
- uris.push_back(encodedUri.toStdString());
+ ret.push_back(encodedUri.toStdString());
}
}
- if (!uris.empty())
- session_.setTurnServers(uris);
+ return ret;
}
-
-void
-CallManager::playRingtone(const QString &ringtone, bool repeat)
-{
- static QMediaPlaylist playlist;
- playlist.clear();
- playlist.setPlaybackMode(repeat ? QMediaPlaylist::CurrentItemInLoop : QMediaPlaylist::CurrentItemOnce);
- playlist.addMedia(QUrl(ringtone));
- player_.setVolume(100);
- player_.setPlaylist(&playlist);
}
-void
-CallManager::stopRingtone()
-{
- player_.setPlaylist(nullptr);
-}
diff --git a/src/CallManager.h b/src/CallManager.h
index 3debf2e85..6518fd133 100644
--- a/src/CallManager.h
+++ b/src/CallManager.h
@@ -11,7 +11,10 @@
#include "mtx/events/collections.hpp"
#include "mtx/events/voip.hpp"
-#include "mtx/responses/turn_server.hpp"
+
+namespace mtx::responses {
+struct TurnServer;
+}
class UserSettings;
class WebRTCSession;
@@ -51,7 +54,7 @@ private:
std::string callid_;
const uint32_t timeoutms_ = 120000;
std::vector<mtx::events::msg::CallCandidates::Candidate> remoteICECandidates_;
- mtx::responses::TurnServer turnServer_;
+ std::vector<std::string> turnURIs_;
QTimer turnServerTimer_;
QSharedPointer<UserSettings> settings_;
QMediaPlayer player_;
@@ -65,7 +68,6 @@ private:
void answerInvite(const mtx::events::msg::CallInvite&);
void generateCallID();
void endCall();
- void setTurnServers();
void playRingtone(const QString &ringtone, bool repeat);
void stopRingtone();
};
diff --git a/src/ChatPage.cpp b/src/ChatPage.cpp
index 5b8ea4752..b53a57617 100644
--- a/src/ChatPage.cpp
+++ b/src/ChatPage.cpp
@@ -137,15 +137,6 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
activeCallBar_->hide();
connect(
&callManager_, &CallManager::newCallParty, activeCallBar_, &ActiveCallBar::setCallParty);
- connect(&WebRTCSession::instance(),
- &WebRTCSession::stateChanged,
- this,
- [this](WebRTCSession::State state) {
- if (state == WebRTCSession::State::DISCONNECTED)
- activeCallBar_->hide();
- else
- activeCallBar_->show();
- });
// Splitter
splitter->addWidget(sideBar_);
diff --git a/src/TextInputWidget.cpp b/src/TextInputWidget.cpp
index d49fc746c..9aadc1010 100644
--- a/src/TextInputWidget.cpp
+++ b/src/TextInputWidget.cpp
@@ -666,7 +666,8 @@ void
TextInputWidget::changeCallButtonState(WebRTCSession::State state)
{
QIcon icon;
- if (state == WebRTCSession::State::DISCONNECTED) {
+ if (state == WebRTCSession::State::ICEFAILED ||
+ state == WebRTCSession::State::DISCONNECTED) {
callBtn_->setToolTip(tr("Place a call"));
icon.addFile(":/icons/icons/ui/place-call.png");
} else {
diff --git a/src/WebRTCSession.cpp b/src/WebRTCSession.cpp
index ff9ec6611..95a9041e3 100644
--- a/src/WebRTCSession.cpp
+++ b/src/WebRTCSession.cpp
@@ -14,9 +14,9 @@ extern "C" {
Q_DECLARE_METATYPE(WebRTCSession::State)
namespace {
-bool gisoffer;
-std::string glocalsdp;
-std::vector<mtx::events::msg::CallCandidates::Candidate> gcandidates;
+bool isoffering_;
+std::string localsdp_;
+std::vector<mtx::events::msg::CallCandidates::Candidate> localcandidates_;
gboolean newBusMessage(GstBus *bus G_GNUC_UNUSED, GstMessage *msg, gpointer user_data);
GstWebRTCSessionDescription* parseSDP(const std::string &sdp, GstWebRTCSDPType type);
@@ -24,6 +24,7 @@ void generateOffer(GstElement *webrtc);
void setLocalDescription(GstPromise *promise, gpointer webrtc);
void addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *candidate, gpointer G_GNUC_UNUSED);
gboolean onICEGatheringCompletion(gpointer timerid);
+void iceConnectionStateChanged(GstElement *webrtcbin, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED);
void createAnswer(GstPromise *promise, gpointer webrtc);
void addDecodeBin(GstElement *webrtc G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
void linkNewPad(GstElement *decodebin G_GNUC_UNUSED, GstPad *newpad, GstElement *pipe);
@@ -92,9 +93,9 @@ WebRTCSession::init(std::string *errorMessage)
bool
WebRTCSession::createOffer()
{
- gisoffer = true;
- glocalsdp.clear();
- gcandidates.clear();
+ isoffering_ = true;
+ localsdp_.clear();
+ localcandidates_.clear();
return startPipeline(111); // a dynamic opus payload type
}
@@ -105,9 +106,9 @@ WebRTCSession::acceptOffer(const std::string &sdp)
if (state_ != State::DISCONNECTED)
return false;
- gisoffer = false;
- glocalsdp.clear();
- gcandidates.clear();
+ isoffering_ = false;
+ localsdp_.clear();
+ localcandidates_.clear();
int opusPayloadType = getPayloadType(sdp, "opus");
if (opusPayloadType == -1)
@@ -152,14 +153,20 @@ WebRTCSession::startPipeline(int opusPayloadType)
gboolean udata;
g_signal_emit_by_name(webrtc_, "add-turn-server", uri.c_str(), (gpointer)(&udata));
}
+ if (turnServers_.empty())
+ nhlog::ui()->warn("WebRTC: no TURN server provided");
// generate the offer when the pipeline goes to PLAYING
- if (gisoffer)
+ if (isoffering_)
g_signal_connect(webrtc_, "on-negotiation-needed", G_CALLBACK(generateOffer), nullptr);
// on-ice-candidate is emitted when a local ICE candidate has been gathered
g_signal_connect(webrtc_, "on-ice-candidate", G_CALLBACK(addLocalICECandidate), nullptr);
+ // capture ICE failure
+ g_signal_connect(webrtc_, "notify::ice-connection-state",
+ G_CALLBACK(iceConnectionStateChanged), nullptr);
+
// incoming streams trigger pad-added
gst_element_set_state(pipe_, GST_STATE_READY);
g_signal_connect(webrtc_, "pad-added", G_CALLBACK(addDecodeBin), pipe_);
@@ -229,8 +236,6 @@ WebRTCSession::acceptICECandidates(const std::vector<mtx::events::msg::CallCandi
nhlog::ui()->debug("WebRTC: remote candidate: (m-line:{}):{}", c.sdpMLineIndex, c.candidate);
g_signal_emit_by_name(webrtc_, "add-ice-candidate", c.sdpMLineIndex, c.candidate.c_str());
}
- if (state_ == State::OFFERSENT)
- emit stateChanged(State::CONNECTING);
}
}
@@ -357,11 +362,11 @@ setLocalDescription(GstPromise *promise, gpointer webrtc)
g_signal_emit_by_name(webrtc, "set-local-description", gstsdp, nullptr);
gchar *sdp = gst_sdp_message_as_text(gstsdp->sdp);
- glocalsdp = std::string(sdp);
+ localsdp_ = std::string(sdp);
g_free(sdp);
gst_webrtc_session_description_free(gstsdp);
- nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", glocalsdp);
+ nhlog::ui()->debug("WebRTC: local description set ({}):\n{}", isAnswer ? "answer" : "offer", localsdp_);
}
void
@@ -369,12 +374,12 @@ addLocalICECandidate(GstElement *webrtc G_GNUC_UNUSED, guint mlineIndex, gchar *
{
nhlog::ui()->debug("WebRTC: local candidate: (m-line:{}):{}", mlineIndex, candidate);
- if (WebRTCSession::instance().state() == WebRTCSession::State::CONNECTED) {
+ if (WebRTCSession::instance().state() >= WebRTCSession::State::OFFERSENT) {
emit WebRTCSession::instance().newICECandidate({"audio", (uint16_t)mlineIndex, candidate});
return;
}
- gcandidates.push_back({"audio", (uint16_t)mlineIndex, candidate});
+ localcandidates_.push_back({"audio", (uint16_t)mlineIndex, candidate});
// GStreamer v1.16: webrtcbin's notify::ice-gathering-state triggers GST_WEBRTC_ICE_GATHERING_STATE_COMPLETE too early
// fixed in v1.18
@@ -390,18 +395,36 @@ gboolean
onICEGatheringCompletion(gpointer timerid)
{
*(guint*)(timerid) = 0;
- if (gisoffer) {
- emit WebRTCSession::instance().offerCreated(glocalsdp, gcandidates);
+ if (isoffering_) {
+ emit WebRTCSession::instance().offerCreated(localsdp_, localcandidates_);
emit WebRTCSession::instance().stateChanged(WebRTCSession::State::OFFERSENT);
}
else {
- emit WebRTCSession::instance().answerCreated(glocalsdp, gcandidates);
- emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
+ emit WebRTCSession::instance().answerCreated(localsdp_, localcandidates_);
+ emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ANSWERSENT);
}
-
return FALSE;
}
+void
+iceConnectionStateChanged(GstElement *webrtc, GParamSpec *pspec G_GNUC_UNUSED, gpointer user_data G_GNUC_UNUSED)
+{
+ GstWebRTCICEConnectionState newState;
+ g_object_get(webrtc, "ice-connection-state", &newState, nullptr);
+ switch (newState) {
+ case GST_WEBRTC_ICE_CONNECTION_STATE_CHECKING:
+ nhlog::ui()->debug("WebRTC: GstWebRTCICEConnectionState -> Checking");
+ emit WebRTCSession::instance().stateChanged(WebRTCSession::State::CONNECTING);
+ break;
+ case GST_WEBRTC_ICE_CONNECTION_STATE_FAILED:
+ nhlog::ui()->error("WebRTC: GstWebRTCICEConnectionState -> Failed");
+ emit WebRTCSession::instance().stateChanged(WebRTCSession::State::ICEFAILED);
+ break;
+ default:
+ break;
+ }
+}
+
void
createAnswer(GstPromise *promise, gpointer webrtc)
{
diff --git a/src/WebRTCSession.h b/src/WebRTCSession.h
index f98820898..d79047a81 100644
--- a/src/WebRTCSession.h
+++ b/src/WebRTCSession.h
@@ -15,10 +15,12 @@ class WebRTCSession : public QObject
public:
enum class State {
+ ICEFAILED,
DISCONNECTED,
INITIATING,
INITIATED,
OFFERSENT,
+ ANSWERSENT,
CONNECTING,
CONNECTED
};
@@ -30,13 +32,13 @@ public:
}
bool init(std::string *errorMessage = nullptr);
+ State state() const {return state_;}
bool createOffer();
bool acceptOffer(const std::string &sdp);
bool acceptAnswer(const std::string &sdp);
void acceptICECandidates(const std::vector<mtx::events::msg::CallCandidates::Candidate>&);
- State state() const {return state_;}
bool toggleMuteAudioSrc(bool &isMuted);
void end();
--
GitLab