Skip to content
Snippets Groups Projects
Commit 4344b696 authored by Konstantinos Sideris's avatar Konstantinos Sideris
Browse files

Save timeline messages in cache for faster startup times

parent 1d6746e4
No related branches found
No related tags found
No related merge requests found
......@@ -37,7 +37,7 @@ set(BOOST_SHA256
5721818253e6a0989583192f96782c4a98eb6204965316df9f5ad75819225ca9)
set(MATRIX_STRUCTS_URL https://github.com/mujx/matrix-structs)
set(MATRIX_STRUCTS_TAG c24cb9b38312dfa24b33413847e3238600c678cd)
set(MATRIX_STRUCTS_TAG 3a052a95c555ce3ae12b8a2e0508e8bb73266fa1)
set(MTXCLIENT_URL https://github.com/mujx/mtxclient)
set(MTXCLIENT_TAG 73491268f94ddeb606284836bb5f512d11b0e249)
......
......@@ -19,8 +19,10 @@
#include <boost/optional.hpp>
#include <QDateTime>
#include <QDir>
#include <QImage>
#include <QString>
#include <json.hpp>
#include <lmdb++.h>
......@@ -46,9 +48,24 @@ struct SearchResult
QString display_name;
};
inline int
numeric_key_comparison(const MDB_val *a, const MDB_val *b)
{
auto lhs = std::stoul(std::string((char *)a->mv_data, a->mv_size));
auto rhs = std::stoul(std::string((char *)b->mv_data, b->mv_size));
if (lhs < rhs)
return 1;
else if (lhs == rhs)
return 0;
return -1;
}
Q_DECLARE_METATYPE(SearchResult)
Q_DECLARE_METATYPE(QVector<SearchResult>)
Q_DECLARE_METATYPE(RoomMember)
Q_DECLARE_METATYPE(mtx::responses::Timeline)
//! Used to uniquely identify a list of read receipts.
struct ReadReceiptKey
......@@ -70,6 +87,15 @@ from_json(const json &j, ReadReceiptKey &key)
key.room_id = j.at("room_id").get<std::string>();
}
struct DescInfo
{
QString username;
QString userid;
QString body;
QString timestamp;
QDateTime datetime;
};
//! UI info associated with a room.
struct RoomInfo
{
......@@ -86,6 +112,8 @@ struct RoomInfo
//! Who can access to the room.
JoinRule join_rule = JoinRule::Public;
bool guest_access = false;
//! Metadata describing the last message in the timeline.
DescInfo msgInfo;
};
inline void
......@@ -289,6 +317,8 @@ public:
bool isFormatValid();
void setCurrentFormat();
std::map<QString, mtx::responses::Timeline> roomMessages();
//! Retrieve all the user ids from a room.
std::vector<std::string> roomMembers(const std::string &room_id);
......@@ -402,6 +432,13 @@ private:
QString getInviteRoomTopic(lmdb::txn &txn, lmdb::dbi &statesdb);
QString getInviteRoomAvatarUrl(lmdb::txn &txn, lmdb::dbi &statesdb, lmdb::dbi &membersdb);
DescInfo getLastMessageInfo(lmdb::txn &txn, const std::string &room_id);
void saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res);
mtx::responses::Timeline getTimelineMessages(lmdb::txn &txn, const std::string &room_id);
//! Remove a room from the cache.
// void removeLeftRoom(lmdb::txn &txn, const std::string &room_id);
template<class T>
......@@ -500,6 +537,7 @@ private:
mpark::holds_alternative<StateEvent<HistoryVisibility>>(e) ||
mpark::holds_alternative<StateEvent<JoinRules>>(e) ||
mpark::holds_alternative<StateEvent<Name>>(e) ||
mpark::holds_alternative<StateEvent<Member>>(e) ||
mpark::holds_alternative<StateEvent<PowerLevels>>(e) ||
mpark::holds_alternative<StateEvent<Topic>>(e);
}
......@@ -544,6 +582,15 @@ private:
}
}
lmdb::dbi getMessagesDb(lmdb::txn &txn, const std::string &room_id)
{
auto db =
lmdb::dbi::open(txn, std::string(room_id + "/messages").c_str(), MDB_CREATE);
lmdb::dbi_set_compare(txn, db, numeric_key_comparison);
return db;
}
lmdb::dbi getInviteStatesDb(lmdb::txn &txn, const std::string &room_id)
{
return lmdb::dbi::open(
......
......@@ -108,7 +108,6 @@ signals:
void showLoginPage(const QString &msg);
void showUserSettingsPage();
void showOverlayProgressBar();
void startConsesusTimer();
void removeTimelineEvent(const QString &room_id, const QString &event_id);
......@@ -124,7 +123,7 @@ signals:
void initializeRoomList(QMap<QString, RoomInfo>);
void initializeViews(const mtx::responses::Rooms &rooms);
void initializeEmptyViews(const std::vector<std::string> &rooms);
void initializeEmptyViews(const std::map<QString, mtx::responses::Timeline> &msgs);
void syncUI(const mtx::responses::Rooms &rooms);
void syncRoomlist(const std::map<QString, RoomInfo> &updates);
void syncTopBar(const std::map<QString, RoomInfo> &updates);
......@@ -206,9 +205,6 @@ private:
TextInputWidget *text_input_;
TypingDisplay *typingDisplay_;
// Safety net if consensus is not possible or too slow.
QTimer *showContentTimer_;
QTimer *consensusTimer_;
QTimer connectivityTimer_;
std::atomic_bool isConnected_;
......
......@@ -22,20 +22,11 @@
#include <QSharedPointer>
#include <QWidget>
#include "Cache.h"
#include <mtx/responses.hpp>
class Menu;
class RippleOverlay;
struct RoomInfo;
struct DescInfo
{
QString username;
QString userid;
QString body;
QString timestamp;
QDateTime datetime;
};
class RoomInfoListItem : public QWidget
{
......
......@@ -41,14 +41,15 @@ template<class T>
QString
messageDescription(const QString &username = "", const QString &body = "")
{
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Sticker = mtx::events::Sticker;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Sticker = mtx::events::Sticker;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Encrypted = mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>;
if (std::is_same<T, AudioItem>::value || std::is_same<T, Audio>::value)
return QString("sent an audio clip");
......@@ -66,6 +67,8 @@ messageDescription(const QString &username = "", const QString &body = "")
return QString(": %1").arg(body);
else if (std::is_same<T, Emote>::value)
return QString("* %1 %2").arg(username).arg(body);
else if (std::is_same<T, Encrypted>::value)
return QString("sent an encrypted message");
}
template<class T, class Event>
......@@ -135,6 +138,18 @@ erase_if(ContainerT &items, const PredicateT &predicate)
}
}
inline uint64_t
event_timestamp(const mtx::events::collections::TimelineEvents &event)
{
return mpark::visit([](auto msg) { return msg.origin_server_ts; }, event);
}
inline nlohmann::json
serialize_event(const mtx::events::collections::TimelineEvents &event)
{
return mpark::visit([](auto msg) { return json(msg); }, event);
}
inline mtx::events::EventType
event_type(const mtx::events::collections::TimelineEvents &event)
{
......
......@@ -158,6 +158,7 @@ public:
//! Remove an item from the timeline with the given Event ID.
void removeEvent(const QString &event_id);
void setPrevBatchToken(const QString &token) { prev_batch_token_ = token; }
public slots:
void sliderRangeChanged(int min, int max);
......
......@@ -27,6 +27,7 @@ class QFile;
class RoomInfoListItem;
class TimelineView;
struct DescInfo;
struct SavedMessages;
class TimelineViewManager : public QStackedWidget
{
......@@ -57,6 +58,7 @@ signals:
public slots:
void removeTimelineEvent(const QString &room_id, const QString &event_id);
void initWithMessages(const std::map<QString, mtx::responses::Timeline> &msgs);
void setHistoryView(const QString &room_id);
void queueTextMessage(const QString &msg);
......
......@@ -21,8 +21,10 @@
#include <QByteArray>
#include <QFile>
#include <QHash>
#include <QSettings>
#include <QStandardPaths>
#include <mtx/responses/common.hpp>
#include <variant.hpp>
#include "Cache.h"
......@@ -38,6 +40,8 @@ static const lmdb::val NEXT_BATCH_KEY("next_batch");
static const lmdb::val OLM_ACCOUNT_KEY("olm_account");
static const lmdb::val CACHE_FORMAT_VERSION_KEY("cache_format_version");
constexpr size_t MAX_RESTORED_MESSAGES = 30;
//! Cache databases and their format.
//!
//! Contains UI information for the joined rooms. (i.e name, topic, avatar url etc).
......@@ -85,6 +89,7 @@ init(const QString &user_id)
qRegisterMetaType<RoomInfo>();
qRegisterMetaType<QMap<QString, RoomInfo>>();
qRegisterMetaType<std::map<QString, RoomInfo>>();
qRegisterMetaType<std::map<QString, mtx::responses::Timeline>>();
instance_ = std::make_unique<Cache>(user_id);
}
......@@ -744,6 +749,8 @@ Cache::saveState(const mtx::responses::Sync &res)
saveStateEvents(txn, statesdb, membersdb, room.first, room.second.state.events);
saveStateEvents(txn, statesdb, membersdb, room.first, room.second.timeline.events);
saveTimelineMessages(txn, room.first, room.second.timeline);
RoomInfo updatedInfo;
updatedInfo.name = getRoomName(txn, statesdb, membersdb).toStdString();
updatedInfo.topic = getRoomTopic(txn, statesdb).toStdString();
......@@ -944,6 +951,57 @@ Cache::getRoomInfo(const std::vector<std::string> &rooms)
return room_info;
}
std::map<QString, mtx::responses::Timeline>
Cache::roomMessages()
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
std::map<QString, mtx::responses::Timeline> msgs;
std::string room_id, unused;
auto roomsCursor = lmdb::cursor::open(txn, roomsDb_);
while (roomsCursor.get(room_id, unused, MDB_NEXT))
msgs.emplace(QString::fromStdString(room_id), getTimelineMessages(txn, room_id));
roomsCursor.close();
txn.commit();
return msgs;
}
mtx::responses::Timeline
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id)
{
auto db = getMessagesDb(txn, room_id);
mtx::responses::Timeline timeline;
std::string timestamp, msg;
auto cursor = lmdb::cursor::open(txn, db);
size_t index = 0;
while (cursor.get(timestamp, msg, MDB_NEXT) && index < MAX_RESTORED_MESSAGES) {
auto obj = json::parse(msg);
if (obj.count("event") == 0 || obj.count("token") == 0)
continue;
mtx::events::collections::TimelineEvents event;
mtx::events::collections::from_json(obj.at("event"), event);
index += 1;
timeline.events.push_back(event);
timeline.prev_batch = obj.at("token").get<std::string>();
}
cursor.close();
std::reverse(timeline.events.begin(), timeline.events.end());
return timeline;
}
QMap<QString, RoomInfo>
Cache::roomInfo(bool withInvites)
{
......@@ -959,6 +1017,8 @@ Cache::roomInfo(bool withInvites)
while (roomsCursor.get(room_id, room_data, MDB_NEXT)) {
RoomInfo tmp = json::parse(std::move(room_data));
tmp.member_count = getMembersDb(txn, room_id).size(txn);
tmp.msgInfo = getLastMessageInfo(txn, room_id);
result.insert(QString::fromStdString(std::move(room_id)), std::move(tmp));
}
roomsCursor.close();
......@@ -979,6 +1039,38 @@ Cache::roomInfo(bool withInvites)
return result;
}
DescInfo
Cache::getLastMessageInfo(lmdb::txn &txn, const std::string &room_id)
{
auto db = getMessagesDb(txn, room_id);
if (db.size(txn) == 0)
return DescInfo{};
std::string timestamp, msg;
QSettings settings;
auto local_user = settings.value("auth/user_id").toString();
auto cursor = lmdb::cursor::open(txn, db);
while (cursor.get(timestamp, msg, MDB_NEXT)) {
auto obj = json::parse(msg);
if (obj.count("event") == 0)
continue;
mtx::events::collections::TimelineEvents event;
mtx::events::collections::from_json(obj.at("event"), event);
cursor.close();
return utils::getMessageDescription(
event, local_user, QString::fromStdString(room_id));
}
cursor.close();
return DescInfo{};
}
std::map<QString, bool>
Cache::invites()
{
......@@ -1512,6 +1604,35 @@ Cache::getMembers(const std::string &room_id, std::size_t startIndex, std::size_
return members;
}
void
Cache::saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res)
{
auto db = getMessagesDb(txn, room_id);
using namespace mtx::events;
using namespace mtx::events::state;
for (const auto &e : res.events) {
if (isStateEvent(e))
continue;
if (mpark::holds_alternative<RedactionEvent<msg::Redaction>>(e))
continue;
json obj = json::object();
obj["event"] = utils::serialize_event(e);
obj["token"] = res.prev_batch;
lmdb::dbi_put(txn,
db,
lmdb::val(std::to_string(utils::event_timestamp(e))),
lmdb::val(obj.dump()));
}
}
void
Cache::markSentNotification(const std::string &event_id)
{
......
......@@ -516,23 +516,6 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
connect(this, &ChatPage::leftRoom, this, &ChatPage::removeRoom);
connect(this, &ChatPage::notificationsRetrieved, this, &ChatPage::sendDesktopNotifications);
showContentTimer_ = new QTimer(this);
showContentTimer_->setSingleShot(true);
connect(showContentTimer_, &QTimer::timeout, this, [this]() {
consensusTimer_->stop();
emit contentLoaded();
});
consensusTimer_ = new QTimer(this);
connect(consensusTimer_, &QTimer::timeout, this, [this]() {
if (view_manager_->hasLoaded()) {
// Remove the spinner overlay.
emit contentLoaded();
showContentTimer_->stop();
consensusTimer_->stop();
}
});
connect(communitiesList_,
&CommunitiesList::communityChanged,
this,
......@@ -552,20 +535,15 @@ ChatPage::ChatPage(QSharedPointer<UserSettings> userSettings, QWidget *parent)
this,
&ChatPage::setGroupViewState);
connect(this, &ChatPage::startConsesusTimer, this, [this]() {
consensusTimer_->start(CONSENSUS_TIMEOUT);
showContentTimer_->start(SHOW_CONTENT_TIMEOUT);
});
connect(this, &ChatPage::initializeRoomList, room_list_, &RoomList::initialize);
connect(this,
&ChatPage::initializeViews,
view_manager_,
[this](const mtx::responses::Rooms &rooms) { view_manager_->initialize(rooms); });
connect(
this,
&ChatPage::initializeEmptyViews,
this,
[this](const std::vector<std::string> &rooms) { view_manager_->initialize(rooms); });
connect(this,
&ChatPage::initializeEmptyViews,
view_manager_,
&TimelineViewManager::initWithMessages);
connect(this, &ChatPage::syncUI, this, [this](const mtx::responses::Rooms &rooms) {
try {
room_list_->cleanupInvites(cache::client()->invites());
......@@ -817,6 +795,8 @@ ChatPage::showUnreadMessageNotification(int count)
void
ChatPage::loadStateFromCache()
{
emit contentLoaded();
nhlog::db()->info("restoring state from cache");
getProfileInfo();
......@@ -829,8 +809,9 @@ ChatPage::loadStateFromCache()
cache::client()->populateMembers();
emit initializeEmptyViews(cache::client()->joinedRooms());
emit initializeEmptyViews(cache::client()->roomMessages());
emit initializeRoomList(cache::client()->roomInfo());
} catch (const mtx::crypto::olm_exception &e) {
nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
emit dropToLoginPageCb(
......@@ -841,6 +822,9 @@ ChatPage::loadStateFromCache()
emit dropToLoginPageCb(
tr("Failed to restore save data. Please login again."));
return;
} catch (const json::exception &e) {
nhlog::db()->critical("failed to parse cache data: {}", e.what());
return;
}
nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
......@@ -848,9 +832,6 @@ ChatPage::loadStateFromCache()
// Start receiving events.
emit trySyncCb();
// Check periodically if the timelines have been loaded.
emit startConsesusTimer();
});
}
......
......@@ -234,7 +234,8 @@ MainWindow::showChatPage()
showOverlayProgressBar();
QTimer::singleShot(100, this, [this]() { pageStack_->setCurrentWidget(chat_page_); });
welcome_page_->hide();
pageStack_->setCurrentWidget(chat_page_);
login_page_->reset();
chat_page_->bootstrap(userid, homeserver, token);
......
......@@ -15,6 +15,7 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <QApplication>
#include <QBuffer>
#include <QObject>
#include <QTimer>
......@@ -171,6 +172,8 @@ RoomList::initialize(const QMap<QString, RoomInfo> &info)
rooms_.clear();
setUpdatesEnabled(false);
for (auto it = info.begin(); it != info.end(); it++) {
if (it.value().is_invite)
addInvitedRoom(it.key(), it.value());
......@@ -178,6 +181,11 @@ RoomList::initialize(const QMap<QString, RoomInfo> &info)
addRoom(it.key(), it.value());
}
for (auto it = info.begin(); it != info.end(); it++)
updateRoomDescription(it.key(), it.value().msgInfo);
setUpdatesEnabled(true);
if (rooms_.empty())
return;
......
......@@ -28,13 +28,14 @@ utils::getMessageDescription(const TimelineEvent &event,
const QString &localUser,
const QString &room_id)
{
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Audio = mtx::events::RoomEvent<mtx::events::msg::Audio>;
using Emote = mtx::events::RoomEvent<mtx::events::msg::Emote>;
using File = mtx::events::RoomEvent<mtx::events::msg::File>;
using Image = mtx::events::RoomEvent<mtx::events::msg::Image>;
using Notice = mtx::events::RoomEvent<mtx::events::msg::Notice>;
using Text = mtx::events::RoomEvent<mtx::events::msg::Text>;
using Video = mtx::events::RoomEvent<mtx::events::msg::Video>;
using Encrypted = mtx::events::EncryptedEvent<mtx::events::msg::Encrypted>;
if (mpark::holds_alternative<Audio>(event)) {
return createDescriptionInfo<Audio>(event, localUser, room_id);
......@@ -52,6 +53,27 @@ utils::getMessageDescription(const TimelineEvent &event,
return createDescriptionInfo<Video>(event, localUser, room_id);
} else if (mpark::holds_alternative<mtx::events::Sticker>(event)) {
return createDescriptionInfo<mtx::events::Sticker>(event, localUser, room_id);
} else if (mpark::holds_alternative<Encrypted>(event)) {
const auto msg = mpark::get<Encrypted>(event);
const auto sender = QString::fromStdString(msg.sender);
const auto username = Cache::displayName(room_id, sender);
const auto ts = QDateTime::fromMSecsSinceEpoch(msg.origin_server_ts);
DescInfo info;
if (sender == localUser)
info.username = "You";
else
info.username = username;
info.userid = sender;
info.body = QString(" %1").arg(messageDescription<Encrypted>());
info.timestamp = utils::descriptiveTime(ts);
info.datetime = ts;
return info;
} else {
std::cout << "type not found: " << serialize_event(event).dump(2) << '\n';
}
return DescInfo{};
......
......@@ -378,7 +378,7 @@ TimelineView::renderBottomEvents(const std::vector<TimelineEvent> &events)
// Prevent blocking of the event-loop
// by calling processEvents every 10 items we render.
if (counter % 10 == 0)
if (counter % 4 == 0)
QApplication::processEvents();
}
}
......@@ -1035,7 +1035,8 @@ TimelineEvent
TimelineView::findLastViewableEvent(const std::vector<TimelineEvent> &events)
{
auto it = std::find_if(events.rbegin(), events.rend(), [](const auto &event) {
return mtx::events::EventType::RoomMessage == utils::event_type(event);
return (mtx::events::EventType::RoomMessage == utils::event_type(event)) ||
(mtx::events::EventType::RoomEncrypted == utils::event_type(event));
});
return (it == std::rend(events)) ? events.back() : *it;
......
......@@ -21,6 +21,7 @@
#include <QFileInfo>
#include <QSettings>
#include "Cache.h"
#include "Logging.hpp"
#include "timeline/TimelineView.h"
#include "timeline/TimelineViewManager.h"
......@@ -146,6 +147,27 @@ TimelineViewManager::initialize(const mtx::responses::Rooms &rooms)
sync(rooms);
}
void
TimelineViewManager::initWithMessages(const std::map<QString, mtx::responses::Timeline> &msgs)
{
for (auto it = msgs.cbegin(); it != msgs.cend(); ++it) {
if (timelineViewExists(it->first))
return;
// Create a history view with the room events.
TimelineView *view = new TimelineView(it->second, it->first);
views_.emplace(it->first, QSharedPointer<TimelineView>(view));
connect(view,
&TimelineView::updateLastTimelineMessage,
this,
&TimelineViewManager::updateRoomsLastMessage);
// Add the view in the widget stack.
addWidget(view);
}
}
void
TimelineViewManager::initialize(const std::vector<std::string> &rooms)
{
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment