From fd83858715dfde7ad61284200821c0357d68ebf8 Mon Sep 17 00:00:00 2001
From: Nicolas Werner <nicolas.werner@hotmail.de>
Date: Mon, 21 Mar 2022 00:48:27 +0100
Subject: [PATCH] Add duration and resolution to files
---
resources/qml/MessageView.qml | 2 +
resources/qml/TimelineRow.qml | 3 +
resources/qml/delegates/MessageDelegate.qml | 3 +
.../qml/delegates/PlayableMediaMessage.qml | 3 +-
resources/qml/delegates/Reply.qml | 2 +
resources/qml/ui/media/MediaControls.qml | 2 +-
src/EventAccessors.cpp | 19 ++++
src/EventAccessors.h | 2 +
src/timeline/InputBar.cpp | 98 ++++++++++++++++-
src/timeline/InputBar.h | 100 +++++++++++++++++-
src/timeline/TimelineModel.cpp | 4 +
src/timeline/TimelineModel.h | 1 +
12 files changed, 230 insertions(+), 9 deletions(-)
diff --git a/resources/qml/MessageView.qml b/resources/qml/MessageView.qml
index 4fce9a759..bbe61ee96 100644
--- a/resources/qml/MessageView.qml
+++ b/resources/qml/MessageView.qml
@@ -376,6 +376,7 @@ Item {
required property string filesize
required property string url
required property string thumbnailUrl
+ required property string duration
required property bool isOnlyEmoji
required property bool isSender
required property bool isEncrypted
@@ -492,6 +493,7 @@ Item {
filesize: wrapper.filesize
url: wrapper.url
thumbnailUrl: wrapper.thumbnailUrl
+ duration: wrapper.duration
isOnlyEmoji: wrapper.isOnlyEmoji
isSender: wrapper.isSender
isEncrypted: wrapper.isEncrypted
diff --git a/resources/qml/TimelineRow.qml b/resources/qml/TimelineRow.qml
index bb6514d1a..032821ba0 100644
--- a/resources/qml/TimelineRow.qml
+++ b/resources/qml/TimelineRow.qml
@@ -41,6 +41,7 @@ Item {
required property var reactions
required property int trustlevel
required property int encryptionError
+ required property int duration
required property var timestamp
required property int status
required property int relatedEventCacheBuster
@@ -128,6 +129,7 @@ Item {
userId: r.relatedEventCacheBuster, fromModel(Room.UserId) ?? ""
userName: r.relatedEventCacheBuster, fromModel(Room.UserName) ?? ""
thumbnailUrl: r.relatedEventCacheBuster, fromModel(Room.ThumbnailUrl) ?? ""
+ duration: r.relatedEventCacheBuster, fromModel(Room.Duration) ?? ""
roomTopic: r.relatedEventCacheBuster, fromModel(Room.RoomTopic) ?? ""
roomName: r.relatedEventCacheBuster, fromModel(Room.RoomName) ?? ""
callType: r.relatedEventCacheBuster, fromModel(Room.CallType) ?? ""
@@ -154,6 +156,7 @@ Item {
typeString: r.typeString ?? ""
url: r.url
thumbnailUrl: r.thumbnailUrl
+ duration: r.duration
originalWidth: r.originalWidth
isOnlyEmoji: r.isOnlyEmoji
isStateEvent: r.isStateEvent
diff --git a/resources/qml/delegates/MessageDelegate.qml b/resources/qml/delegates/MessageDelegate.qml
index 08b2098ed..0e211ded3 100644
--- a/resources/qml/delegates/MessageDelegate.qml
+++ b/resources/qml/delegates/MessageDelegate.qml
@@ -18,6 +18,7 @@ Item {
required property int type
required property string typeString
required property int originalWidth
+ required property int duration
required property string blurhash
required property string body
required property string formattedBody
@@ -161,6 +162,7 @@ Item {
url: d.url
body: d.body
filesize: d.filesize
+ duration: d.duration
metadataWidth: d.metadataWidth
}
@@ -178,6 +180,7 @@ Item {
url: d.url
body: d.body
filesize: d.filesize
+ duration: d.duration
metadataWidth: d.metadataWidth
}
diff --git a/resources/qml/delegates/PlayableMediaMessage.qml b/resources/qml/delegates/PlayableMediaMessage.qml
index 5d7beaadd..405727044 100644
--- a/resources/qml/delegates/PlayableMediaMessage.qml
+++ b/resources/qml/delegates/PlayableMediaMessage.qml
@@ -17,6 +17,7 @@ Item {
required property double proportionalHeight
required property int type
required property int originalWidth
+ required property int duration
required property string thumbnailUrl
required property string eventId
required property string url
@@ -85,7 +86,7 @@ Item {
anchors.bottom: fileInfoLabel.top
playingVideo: type == MtxEvent.VideoMessage
positionValue: mxcmedia.position
- duration: mxcmedia.duration
+ duration: mediaLoaded ? mxcmedia.duration : content.duration
mediaLoaded: mxcmedia.loaded
mediaState: mxcmedia.state
onPositionChanged: mxcmedia.position = position
diff --git a/resources/qml/delegates/Reply.qml b/resources/qml/delegates/Reply.qml
index 513b7c0b3..27fb4e070 100644
--- a/resources/qml/delegates/Reply.qml
+++ b/resources/qml/delegates/Reply.qml
@@ -34,6 +34,7 @@ Item {
property string roomTopic
property string roomName
property string callType
+ property int duration
property int encryptionError
property int relatedEventCacheBuster
property int maxWidth
@@ -112,6 +113,7 @@ Item {
typeString: r.typeString ?? ""
url: r.url
thumbnailUrl: r.thumbnailUrl
+ duration: r.duration
originalWidth: r.originalWidth
isOnlyEmoji: r.isOnlyEmoji
isStateEvent: r.isStateEvent
diff --git a/resources/qml/ui/media/MediaControls.qml b/resources/qml/ui/media/MediaControls.qml
index 1844af73d..d73957eef 100644
--- a/resources/qml/ui/media/MediaControls.qml
+++ b/resources/qml/ui/media/MediaControls.qml
@@ -214,7 +214,7 @@ Rectangle {
Label {
Layout.alignment: Qt.AlignRight
- text: (!control.mediaLoaded) ? "-- / --" : (durationToString(control.positionValue) + " / " + durationToString(control.duration))
+ text: (!control.mediaLoaded ? "-- " : durationToString(control.positionValue)) + " / " + durationToString(control.duration)
color: Nheko.colors.text
}
diff --git a/src/EventAccessors.cpp b/src/EventAccessors.cpp
index 935ff73a0..00cea86e3 100644
--- a/src/EventAccessors.cpp
+++ b/src/EventAccessors.cpp
@@ -169,6 +169,20 @@ struct EventThumbnailUrl
}
};
+struct EventDuration
+{
+ template<class Content>
+ using thumbnail_url_t = decltype(Content::info.duration);
+ template<class T>
+ uint64_t operator()(const mtx::events::Event<T> &e)
+ {
+ if constexpr (is_detected<thumbnail_url_t, T>::value) {
+ return e.content.info.duration;
+ }
+ return 0;
+ }
+};
+
struct EventBlurhash
{
template<class Content>
@@ -420,6 +434,11 @@ mtx::accessors::thumbnail_url(const mtx::events::collections::TimelineEvents &ev
{
return std::visit(EventThumbnailUrl{}, event);
}
+uint64_t
+mtx::accessors::duration(const mtx::events::collections::TimelineEvents &event)
+{
+ return std::visit(EventDuration{}, event);
+}
std::string
mtx::accessors::blurhash(const mtx::events::collections::TimelineEvents &event)
{
diff --git a/src/EventAccessors.h b/src/EventAccessors.h
index e46d4786a..a74c58bcb 100644
--- a/src/EventAccessors.h
+++ b/src/EventAccessors.h
@@ -83,6 +83,8 @@ std::string
url(const mtx::events::collections::TimelineEvents &event);
std::string
thumbnail_url(const mtx::events::collections::TimelineEvents &event);
+uint64_t
+duration(const mtx::events::collections::TimelineEvents &event);
std::string
blurhash(const mtx::events::collections::TimelineEvents &event);
std::string
diff --git a/src/timeline/InputBar.cpp b/src/timeline/InputBar.cpp
index 1b7d6efba..eda4507ae 100644
--- a/src/timeline/InputBar.cpp
+++ b/src/timeline/InputBar.cpp
@@ -11,6 +11,8 @@
#include <QFileDialog>
#include <QGuiApplication>
#include <QInputMethod>
+#include <QMediaMetaData>
+#include <QMediaPlayer>
#include <QMimeData>
#include <QMimeDatabase>
#include <QStandardPaths>
@@ -452,7 +454,8 @@ InputBar::audio(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const QString &mime,
- uint64_t dsize)
+ uint64_t dsize,
+ uint64_t duration)
{
mtx::events::msg::Audio audio;
audio.info.mimetype = mime.toStdString();
@@ -460,6 +463,9 @@ InputBar::audio(const QString &filename,
audio.body = filename.toStdString();
audio.url = url.toStdString();
+ if (duration > 0)
+ audio.info.duration = duration;
+
if (file)
audio.file = file;
else
@@ -482,13 +488,22 @@ InputBar::video(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const QString &mime,
- uint64_t dsize)
+ uint64_t dsize,
+ uint64_t duration,
+ const QSize &dimensions)
{
mtx::events::msg::Video video;
video.info.mimetype = mime.toStdString();
video.info.size = dsize;
video.body = filename.toStdString();
+ if (duration > 0)
+ video.info.duration = duration;
+ if (dimensions.isValid()) {
+ video.info.h = dimensions.height();
+ video.info.w = dimensions.width();
+ }
+
if (file)
video.file = file;
else
@@ -645,6 +660,7 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
source->open(QIODevice::ReadOnly);
data = source->readAll();
+ source->reset();
if (!data.size()) {
nhlog::ui()->warn("Attempted to upload zero-byte file?! Mimetype {}, filename {}",
@@ -657,6 +673,8 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
nhlog::ui()->debug("Mime: {}", mimetype_.toStdString());
if (mimeClass_ == u"image") {
QImage img = utils::readImage(data);
+ setThumbnail(img.scaled(
+ std::min(800, img.width()), std::min(800, img.height()), Qt::KeepAspectRatioByExpanding));
dimensions_ = img.size();
if (img.height() > 200 && img.width() > 360)
@@ -672,6 +690,78 @@ MediaUpload::MediaUpload(std::unique_ptr<QIODevice> source_,
}
blurhash_ =
QString::fromStdString(blurhash::encode(data_.data(), img.width(), img.height(), 4, 3));
+ } else if (mimeClass_ == u"video" || mimeClass_ == u"audio") {
+ auto mediaPlayer = new QMediaPlayer(
+ this,
+ mimeClass_ == u"video" ? QFlags{QMediaPlayer::StreamPlayback, QMediaPlayer::VideoSurface}
+ : QFlags{QMediaPlayer::StreamPlayback});
+ mediaPlayer->setMuted(true);
+
+ if (mimeClass_ == u"video") {
+ auto newSurface = new InputVideoSurface(this);
+ connect(
+ newSurface, &InputVideoSurface::newImage, this, [this, mediaPlayer](QImage img) {
+ mediaPlayer->stop();
+
+ nhlog::ui()->debug("Got image {}x{}", img.width(), img.height());
+
+ this->setThumbnail(img);
+
+ if (!dimensions_.isValid())
+ this->dimensions_ = img.size();
+
+ if (img.height() > 200 && img.width() > 360)
+ img = img.scaled(360, 200, Qt::KeepAspectRatioByExpanding);
+ std::vector<unsigned char> data_;
+ for (int y = 0; y < img.height(); y++) {
+ for (int x = 0; x < img.width(); x++) {
+ auto p = img.pixel(x, y);
+ data_.push_back(static_cast<unsigned char>(qRed(p)));
+ data_.push_back(static_cast<unsigned char>(qGreen(p)));
+ data_.push_back(static_cast<unsigned char>(qBlue(p)));
+ }
+ }
+ blurhash_ = QString::fromStdString(
+ blurhash::encode(data_.data(), img.width(), img.height(), 4, 3));
+ });
+ mediaPlayer->setVideoOutput(newSurface);
+ }
+
+ connect(mediaPlayer,
+ qOverload<QMediaPlayer::Error>(&QMediaPlayer::error),
+ this,
+ [this, mediaPlayer](QMediaPlayer::Error error) {
+ nhlog::ui()->info("Media player error {} and errorStr {}",
+ error,
+ mediaPlayer->errorString().toStdString());
+ });
+ connect(mediaPlayer,
+ &QMediaPlayer::mediaStatusChanged,
+ [this, mediaPlayer](QMediaPlayer::MediaStatus status) {
+ nhlog::ui()->info(
+ "Media player status {} and error {}", status, mediaPlayer->error());
+ });
+ connect(mediaPlayer,
+ qOverload<const QString &, const QVariant &>(&QMediaPlayer::metaDataChanged),
+ [this, mediaPlayer](QString t, QVariant) {
+ nhlog::ui()->info("Got metadata {}", t.toStdString());
+
+ if (mediaPlayer->duration() > 0)
+ this->duration_ = mediaPlayer->duration();
+
+ dimensions_ = mediaPlayer->metaData(QMediaMetaData::Resolution).toSize();
+ auto orientation = mediaPlayer->metaData(QMediaMetaData::Orientation).toInt();
+ if (orientation == 90 || orientation == 270) {
+ dimensions_.transpose();
+ }
+ });
+ connect(mediaPlayer, &QMediaPlayer::durationChanged, [this, mediaPlayer](qint64 duration) {
+ if (duration > 0)
+ this->duration_ = mediaPlayer->duration();
+ nhlog::ui()->info("Duration changed {}", duration);
+ });
+ mediaPlayer->setMedia(QMediaContent(originalFilename_), source.get());
+ mediaPlayer->play();
}
}
@@ -721,9 +811,9 @@ InputBar::finalizeUpload(MediaUpload *upload, QString url)
if (mimeClass == u"image")
image(filename, encryptedFile, url, mime, size, upload->dimensions(), upload->blurhash());
else if (mimeClass == u"audio")
- audio(filename, encryptedFile, url, mime, size);
+ audio(filename, encryptedFile, url, mime, size, upload->duration());
else if (mimeClass == u"video")
- video(filename, encryptedFile, url, mime, size);
+ video(filename, encryptedFile, url, mime, size, upload->duration(), upload->dimensions());
else
file(filename, encryptedFile, url, mime, size);
diff --git a/src/timeline/InputBar.h b/src/timeline/InputBar.h
index 607736b6e..97d262ccd 100644
--- a/src/timeline/InputBar.h
+++ b/src/timeline/InputBar.h
@@ -5,7 +5,9 @@
#pragma once
+#include <QAbstractVideoSurface>
#include <QIODevice>
+#include <QImage>
#include <QObject>
#include <QSize>
#include <QStringList>
@@ -29,6 +31,90 @@ enum class MarkdownOverride
OFF,
};
+class InputVideoSurface : public QAbstractVideoSurface
+{
+ Q_OBJECT
+
+public:
+ InputVideoSurface(QObject *parent)
+ : QAbstractVideoSurface(parent)
+ {}
+
+ bool present(const QVideoFrame &frame) override
+ {
+ QImage::Format format = QImage::Format_Invalid;
+
+ switch (frame.pixelFormat()) {
+ case QVideoFrame::Format_ARGB32:
+ format = QImage::Format_ARGB32;
+ break;
+ case QVideoFrame::Format_ARGB32_Premultiplied:
+ format = QImage::Format_ARGB32_Premultiplied;
+ break;
+ case QVideoFrame::Format_RGB24:
+ format = QImage::Format_RGB888;
+ break;
+ case QVideoFrame::Format_BGR24:
+ format = QImage::Format_BGR888;
+ break;
+ case QVideoFrame::Format_RGB32:
+ format = QImage::Format_RGB32;
+ break;
+ case QVideoFrame::Format_RGB565:
+ format = QImage::Format_RGB16;
+ break;
+ case QVideoFrame::Format_RGB555:
+ format = QImage::Format_RGB555;
+ break;
+ default:
+ format = QImage::Format_Invalid;
+ }
+
+ if (format == QImage::Format_Invalid) {
+ emit newImage({});
+ return false;
+ } else {
+ QVideoFrame frametodraw(frame);
+
+ if (!frametodraw.map(QAbstractVideoBuffer::ReadOnly)) {
+ emit newImage({});
+ return false;
+ }
+
+ // this is a shallow operation. it just refer the frame buffer
+ QImage image(frametodraw.bits(),
+ frametodraw.width(),
+ frametodraw.height(),
+ frametodraw.bytesPerLine(),
+ QImage::Format_RGB444);
+
+ emit newImage(std::move(image));
+ return true;
+ }
+ }
+
+ QList<QVideoFrame::PixelFormat>
+ supportedPixelFormats(QAbstractVideoBuffer::HandleType type) const override
+ {
+ if (type == QAbstractVideoBuffer::NoHandle) {
+ return {
+ QVideoFrame::Format_ARGB32,
+ QVideoFrame::Format_ARGB32_Premultiplied,
+ QVideoFrame::Format_RGB24,
+ QVideoFrame::Format_BGR24,
+ QVideoFrame::Format_RGB32,
+ QVideoFrame::Format_RGB565,
+ QVideoFrame::Format_RGB555,
+ };
+ } else {
+ return {};
+ }
+ }
+
+signals:
+ void newImage(QImage img);
+};
+
class MediaUpload : public QObject
{
Q_OBJECT
@@ -67,6 +153,7 @@ public:
[[nodiscard]] QString filename() const { return originalFilename_; }
[[nodiscard]] QString blurhash() const { return blurhash_; }
[[nodiscard]] uint64_t size() const { return size_; }
+ [[nodiscard]] uint64_t duration() const { return duration_; }
[[nodiscard]] std::optional<mtx::crypto::EncryptedFile> encryptedFile_()
{
return encryptedFile;
@@ -82,6 +169,7 @@ public slots:
private slots:
void updateThumbnailUrl(QString url) { this->thumbnailUrl_ = std::move(url); }
+ void setThumbnail(QImage img) { this->thumbnail_ = std::move(img); }
public:
// void uploadThumbnail(QImage img);
@@ -96,8 +184,11 @@ public:
QString url_;
std::optional<mtx::crypto::EncryptedFile> encryptedFile;
+ QImage thumbnail_;
+
QSize dimensions_;
- uint64_t size_ = 0;
+ uint64_t size_ = 0;
+ uint64_t duration_ = 0;
bool encrypt_;
};
@@ -181,12 +272,15 @@ private:
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const QString &mime,
- uint64_t dsize);
+ uint64_t dsize,
+ uint64_t duration);
void video(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const QString &mime,
- uint64_t dsize);
+ uint64_t dsize,
+ uint64_t duration,
+ const QSize &dimensions);
void startUploadFromPath(const QString &path);
void startUploadFromMimeData(const QMimeData &source, const QString &format);
diff --git a/src/timeline/TimelineModel.cpp b/src/timeline/TimelineModel.cpp
index 8e6c72355..4c1ce2dc1 100644
--- a/src/timeline/TimelineModel.cpp
+++ b/src/timeline/TimelineModel.cpp
@@ -474,6 +474,7 @@ TimelineModel::roleNames() const
{Timestamp, "timestamp"},
{Url, "url"},
{ThumbnailUrl, "thumbnailUrl"},
+ {Duration, "duration"},
{Blurhash, "blurhash"},
{Filename, "filename"},
{Filesize, "filesize"},
@@ -627,6 +628,8 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r
return QVariant(QString::fromStdString(url(event)));
case ThumbnailUrl:
return QVariant(QString::fromStdString(thumbnail_url(event)));
+ case Duration:
+ return QVariant(static_cast<qulonglong>(duration(event)));
case Blurhash:
return QVariant(QString::fromStdString(blurhash(event)));
case Filename:
@@ -739,6 +742,7 @@ TimelineModel::data(const mtx::events::collections::TimelineEvents &event, int r
m.insert(names[Timestamp], data(event, static_cast<int>(Timestamp)));
m.insert(names[Url], data(event, static_cast<int>(Url)));
m.insert(names[ThumbnailUrl], data(event, static_cast<int>(ThumbnailUrl)));
+ m.insert(names[Duration], data(event, static_cast<int>(Duration)));
m.insert(names[Blurhash], data(event, static_cast<int>(Blurhash)));
m.insert(names[Filename], data(event, static_cast<int>(Filename)));
m.insert(names[Filesize], data(event, static_cast<int>(Filesize)));
diff --git a/src/timeline/TimelineModel.h b/src/timeline/TimelineModel.h
index f47203f0d..7e21a394e 100644
--- a/src/timeline/TimelineModel.h
+++ b/src/timeline/TimelineModel.h
@@ -215,6 +215,7 @@ public:
Timestamp,
Url,
ThumbnailUrl,
+ Duration,
Blurhash,
Filename,
Filesize,
--
GitLab