Cache.cpp

// SPDX-FileCopyrightText: Nheko Contributors
//
// SPDX-License-Identifier: GPL-3.0-or-later

#include "Cache.h"
#include "Cache_p.h"

#include <stdexcept>
#include <unordered_set>
#include <variant>

#include <QCoreApplication>
#include <QCryptographicHash>
#include <QDir>
#include <QFile>
#include <QHash>
#include <QMap>
#include <QMessageBox>
#include <QStandardPaths>

#if __has_include(<keychain.h>)
#include <keychain.h>
#else
#include <qt6keychain/keychain.h>
#endif

#include <nlohmann/json.hpp>

#include <mtx/responses/common.hpp>
#include <mtx/responses/messages.hpp>

#include "ChatPage.h"
#include "EventAccessors.h"
#include "Logging.h"
#include "MatrixClient.h"
#include "UserSettingsPage.h"
#include "Utils.h"
#include "encryption/Olm.h"

//! Should be changed when a breaking change occurs in the cache format.
//! This will reset client's data.
static constexpr std::string_view CURRENT_CACHE_FORMAT_VERSION{"2023.10.22"};
static constexpr std::string_view MAX_DBS_SETTINGS_KEY{"database/maxdbs"};
static constexpr std::string_view MAX_DB_SIZE_SETTINGS_KEY{"database/maxsize"};

//! Keys used for the DB
static const std::string_view NEXT_BATCH_KEY("next_batch");
static const std::string_view OLM_ACCOUNT_KEY("olm_account");
static const std::string_view CACHE_FORMAT_VERSION_KEY("cache_format_version");
static const std::string_view CURRENT_ONLINE_BACKUP_VERSION("current_online_backup_version");

static constexpr auto MAX_DBS_DEFAULT = 32384U;

#if Q_PROCESSOR_WORDSIZE >= 5 // 40-bit or more, up to 2^(8*WORDSIZE) words addressable.
static constexpr auto DB_SIZE_DEFAULT         = 32ULL * 1024ULL * 1024ULL * 1024ULL; // 32 GB
static constexpr size_t MAX_RESTORED_MESSAGES = 30'000;
#elif Q_PROCESSOR_WORDSIZE == 4 // 32-bit address space limits mmaps
static constexpr auto DB_SIZE_DEFAULT         = 1ULL * 1024ULL * 1024ULL * 1024ULL; // 1 GB
static constexpr size_t MAX_RESTORED_MESSAGES = 5'000;
#else
#error Not enough virtual address space for the database on target CPU
#endif

//! Cache databases and their format.
//!
//! Contains UI information for the joined rooms. (i.e name, topic, avatar url etc).
//! Format: room_id -> RoomInfo
static constexpr auto ROOMS_DB("rooms");
static constexpr auto INVITES_DB("invites");
//! maps each room to its parent space (id->id)
static constexpr auto SPACES_PARENTS_DB("space_parents");
//! maps each space to its current children (id->id)
static constexpr auto SPACES_CHILDREN_DB("space_children");
//! Information that  must be kept between sync requests.
static constexpr auto SYNC_STATE_DB("sync_state");
//! Read receipts per room/event.
static constexpr auto READ_RECEIPTS_DB("read_receipts");
static constexpr auto NOTIFICATIONS_DB("sent_notifications");
static constexpr auto PRESENCE_DB("presence");

//! Encryption related databases.

//! user_id -> list of devices
static constexpr auto DEVICES_DB("devices");
//! device_id -> device keys
static constexpr auto DEVICE_KEYS_DB("device_keys");
//! room_ids that have encryption enabled.
static constexpr auto ENCRYPTED_ROOMS_DB("encrypted_rooms");
//! Expiration progress for each room
static constexpr auto EVENT_EXPIRATION_BG_JOB_DB("event_expiration_bg_job");

//! room_id -> pickled OlmInboundGroupSession
static constexpr auto INBOUND_MEGOLM_SESSIONS_DB("inbound_megolm_sessions");
//! MegolmSessionIndex -> pickled OlmOutboundGroupSession
static constexpr auto OUTBOUND_MEGOLM_SESSIONS_DB("outbound_megolm_sessions");
//! MegolmSessionIndex -> session data about which devices have access to this
static constexpr auto MEGOLM_SESSIONS_DATA_DB("megolm_sessions_data_db");
//! Curve25519 key to session_id and json encoded olm session, separated by null. Dupsorted.
static constexpr auto OLM_SESSIONS_DB("olm_sessions.v3");

//! flag to be set, when the db should be compacted on startup
bool needsCompact = false;

using CachedReceipts = std::multimap<uint64_t, std::string, std::greater<uint64_t>>;
using Receipts       = std::map<std::string, std::map<std::string, uint64_t>>;

static std::string
combineOlmSessionKeyFromCurveAndSessionId(std::string_view curve25519, std::string_view session_id)
{
    std::string combined(curve25519.size() + 1 + session_id.size(), '\0');
    combined.replace(0, curve25519.size(), curve25519);
    combined.replace(curve25519.size() + 1, session_id.size(), session_id);
    return combined;
}
static std::pair<std::string_view, std::string_view>
splitCurve25519AndOlmSessionId(std::string_view input)
{
    auto separator = input.find('\0');
    return std::pair(input.substr(0, separator), input.substr(separator + 1));
}

namespace {
std::unique_ptr<Cache> instance_ = nullptr;
}

struct RO_txn
{
    ~RO_txn() { txn.reset(); }
    operator MDB_txn *() const noexcept { return txn.handle(); }
    operator lmdb::txn &() noexcept { return txn; }

    lmdb::txn &txn;
};

RO_txn
ro_txn(lmdb::env &env)
{
    thread_local lmdb::txn txn     = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
    thread_local int reuse_counter = 0;

    if (reuse_counter >= 100 || txn.env() != env.handle()) {
        txn.abort();
        txn           = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
        reuse_counter = 0;
    } else if (reuse_counter > 0) {
        try {
            txn.renew();
        } catch (...) {
            txn.abort();
            txn           = lmdb::txn::begin(env, nullptr, MDB_RDONLY);
            reuse_counter = 0;
        }
    }
    reuse_counter++;

    return RO_txn{txn};
}

static void
compactDatabase(lmdb::env &from, lmdb::env &to)
{
    auto fromTxn = lmdb::txn::begin(from, nullptr, MDB_RDONLY);
    auto toTxn   = lmdb::txn::begin(to);

    auto rootDb  = lmdb::dbi::open(fromTxn);
    auto dbNames = lmdb::cursor::open(fromTxn, rootDb);

    std::string_view dbName;
    while (dbNames.get(dbName, MDB_cursor_op::MDB_NEXT_NODUP)) {
        nhlog::db()->info("Compacting db: {}", dbName);

        auto flags = MDB_CREATE;

        if (dbName.ends_with("/event_order") || dbName.ends_with("/order2msg") ||
            dbName.ends_with("/pending"))
            flags |= MDB_INTEGERKEY;
        if (dbName.ends_with("/related") || dbName.ends_with("/states_key") ||
            dbName == SPACES_CHILDREN_DB || dbName == SPACES_PARENTS_DB)
            flags |= MDB_DUPSORT;

        auto dbNameStr = std::string(dbName);
        auto fromDb    = lmdb::dbi::open(fromTxn, dbNameStr.c_str(), flags);
        auto toDb      = lmdb::dbi::open(toTxn, dbNameStr.c_str(), flags);

        if (dbName.ends_with("/states_key")) {
            lmdb::dbi_set_dupsort(fromTxn, fromDb, Cache::compare_state_key);
            lmdb::dbi_set_dupsort(toTxn, toDb, Cache::compare_state_key);
        }

        auto fromCursor = lmdb::cursor::open(fromTxn, fromDb);
        auto toCursor   = lmdb::cursor::open(toTxn, toDb);

        std::string_view key, val;
        while (fromCursor.get(key, val, MDB_cursor_op::MDB_NEXT)) {
            toCursor.put(key, val, MDB_APPENDDUP);
        }
    }

    toTxn.commit();
}

static bool
containsStateUpdates(const mtx::events::collections::StrippedEvents &e)
{
    using namespace mtx::events;
    using namespace mtx::events::state;

    return std::holds_alternative<StrippedEvent<state::Avatar>>(e) ||
           std::holds_alternative<StrippedEvent<CanonicalAlias>>(e) ||
           std::holds_alternative<StrippedEvent<Name>>(e) ||
           std::holds_alternative<StrippedEvent<Member>>(e) ||
           std::holds_alternative<StrippedEvent<Topic>>(e);
}

bool
Cache::isHiddenEvent(lmdb::txn &txn,
                     mtx::events::collections::TimelineEvents e,
                     const std::string &room_id)
{
    using namespace mtx::events;

    // Always hide edits
    if (mtx::accessors::relations(e).replaces())
        return true;

    if (auto encryptedEvent = std::get_if<EncryptedEvent<msg::Encrypted>>(&e)) {
        MegolmSessionIndex index;
        index.room_id    = room_id;
        index.session_id = encryptedEvent->content.session_id;

        auto result = olm::decryptEvent(index, *encryptedEvent, true);
        if (!result.error)
            e = result.event.value();
    }

    mtx::events::account_data::nheko_extensions::HiddenEvents hiddenEvents;
    hiddenEvents.hidden_event_types = std::vector{
      EventType::Reaction,
      EventType::CallCandidates,
      EventType::CallNegotiate,
      EventType::Unsupported,
    };
    // check if selected answer is from to local user
    /*
     * localUser accepts/rejects the call and it is selected by caller - No message
     * Another User accepts/rejects the call and it is selected by caller - "Call answered/rejected
     * elsewhere"
     */
    bool callLocalUser_ = true;
    if (callLocalUser_)
        hiddenEvents.hidden_event_types->push_back(EventType::CallSelectAnswer);

    if (auto temp = getAccountData(txn, mtx::events::EventType::NhekoHiddenEvents, "")) {
        auto h = std::get<
          mtx::events::AccountDataEvent<mtx::events::account_data::nheko_extensions::HiddenEvents>>(
          *temp);
        if (h.content.hidden_event_types)
            hiddenEvents = std::move(h.content);
    }
    if (auto temp = getAccountData(txn, mtx::events::EventType::NhekoHiddenEvents, room_id)) {
        auto h = std::get<
          mtx::events::AccountDataEvent<mtx::events::account_data::nheko_extensions::HiddenEvents>>(
          *temp);
        if (h.content.hidden_event_types)
            hiddenEvents = std::move(h.content);
    }

    return std::find(hiddenEvents.hidden_event_types->begin(),
                     hiddenEvents.hidden_event_types->end(),
                     std::visit([](const auto &ev) { return ev.type; }, e)) !=
           hiddenEvents.hidden_event_types->end();
}

Cache::Cache(const QString &userId, QObject *parent)
  : QObject{parent}
  , env_{nullptr}
  , localUserId_{userId}
{
    connect(this, &Cache::userKeysUpdate, this, &Cache::updateUserKeys, Qt::QueuedConnection);
    connect(
      this,
      &Cache::verificationStatusChanged,
      this,
      [this](const std::string &u) {
          if (u == localUserId_.toStdString()) {
              auto status = verificationStatus(u);
              emit selfVerificationStatusChanged();
          }
      },
      Qt::QueuedConnection);
    setup();
}

static QString
cacheDirectoryName(const QString &userid, const QString &profile)
{
    QCryptographicHash hash(QCryptographicHash::Algorithm::Sha256);
    hash.addData(userid.toUtf8());
    hash.addData(profile.toUtf8());
    return QStringLiteral("%1/db-%2")
      .arg(QStandardPaths::writableLocation(QStandardPaths::AppDataLocation),
           hash.result().toHex());
}

void
Cache::setup()
{
    auto settings = UserSettings::instance();

    nhlog::db()->debug("setting up cache");

    // Previous location of the cache directory
    auto oldCache2 =
      QStringLiteral("%1/%2%3").arg(QStandardPaths::writableLocation(QStandardPaths::CacheLocation),
                                    QString::fromUtf8(localUserId_.toUtf8().toHex()),
                                    QString::fromUtf8(settings->profile().toUtf8().toHex()));

    auto oldCache = QStringLiteral("%1/%2%3").arg(
      QStandardPaths::writableLocation(QStandardPaths::AppDataLocation),
      QString::fromUtf8(localUserId_.toUtf8().toHex()),
      QString::fromUtf8(settings->profile().toUtf8().toHex()));

    cacheDirectory_ = cacheDirectoryName(localUserId_, settings->profile());

    nhlog::db()->debug("Database at: {}", cacheDirectory_.toStdString());

    bool isInitial = !QFile::exists(cacheDirectory_);

    // NOTE: If both cache directories exist it's better to do nothing: it
    // could mean a previous migration failed or was interrupted.
    if (isInitial) {
        if (QFile::exists(oldCache)) {
            nhlog::db()->info("found old state directory, migrating");
            if (!QDir().rename(oldCache, cacheDirectory_)) {
                throw std::runtime_error(("Unable to migrate the old state directory (" + oldCache +
                                          ") to the new location (" + cacheDirectory_ + ")")
                                           .toStdString()
                                           .c_str());
            }
            nhlog::db()->info("completed state migration");
        } else if (QFile::exists(oldCache2)) {
            nhlog::db()->info("found very old state directory, migrating");
            if (!QDir().rename(oldCache2, cacheDirectory_)) {
                throw std::runtime_error(("Unable to migrate the very old state directory (" +
                                          oldCache2 + ") to the new location (" + cacheDirectory_ +
                                          ")")
                                           .toStdString()
                                           .c_str());
            }
            nhlog::db()->info("completed state migration");
        }
    }

    auto openEnv = [](const QString &name) {
        auto settings      = UserSettings::instance();
        std::size_t dbSize = std::max(
          settings->qsettings()->value(MAX_DB_SIZE_SETTINGS_KEY, DB_SIZE_DEFAULT).toULongLong(),
          DB_SIZE_DEFAULT);
        unsigned dbCount =
          std::max(settings->qsettings()->value(MAX_DBS_SETTINGS_KEY, MAX_DBS_DEFAULT).toUInt(),
                   MAX_DBS_DEFAULT);

        // ignore unreasonably high values of more than a quarter of the addressable memory
        if (dbSize > (1ull << (Q_PROCESSOR_WORDSIZE * 8 - 2))) {
            dbSize = DB_SIZE_DEFAULT;
        }
        // Limit databases to about a million. This would cause more than 7-120MB to get written on
        // every commit, which I doubt would work well. File an issue, if you tested this and it
        // works fine.
        if (dbCount > (1u << 20)) {
            dbCount = 1u << 20;
        }

        auto e = lmdb::env::create();
        e.set_mapsize(dbSize);
        e.set_max_dbs(dbCount);
        e.open(name.toStdString().c_str(), MDB_NOMETASYNC | MDB_NOSYNC);
        return e;
    };

    if (isInitial) {
        nhlog::db()->info("initializing LMDB");

        if (!QDir().mkpath(cacheDirectory_)) {
            throw std::runtime_error(
              ("Unable to create state directory:" + cacheDirectory_).toStdString().c_str());
        }
    }

    try {
        // NOTE(Nico): We may want to use (MDB_MAPASYNC | MDB_WRITEMAP) in the future, but
        // it can really mess up our database, so we shouldn't. For now, hopefully
        // NOMETASYNC is fast enough.
        //
        // 2022-10-28: Disable the nosync flags again in the hope to crack down on some database
        // corruption.
        // 2023-02-23: Reenable the nosync flags. There was no measureable benefit to resiliency,
        // but sync causes frequent lag sometimes even for the whole system. Possibly the data
        // corruption is an lmdb or filesystem bug. See
        // https://github.com/Nheko-Reborn/nheko/issues/1355
        // https://github.com/Nheko-Reborn/nheko/issues/1303
        env_ = openEnv(cacheDirectory_);

        if (needsCompact) {
            auto compactDir  = cacheDirectory_ + "-compacting";
            auto toDeleteDir = cacheDirectory_ + "-olddb";
            if (QFile::exists(cacheDirectory_))
                QDir(compactDir).removeRecursively();
            if (QFile::exists(toDeleteDir))
                QDir(toDeleteDir).removeRecursively();
            if (!QDir().mkpath(compactDir)) {
                nhlog::db()->warn(
                  "Failed to create directory '{}' for database compaction, skipping compaction!",
                  compactDir.toStdString());
            } else {
                // lmdb::env_copy(env_, compactDir.toStdString().c_str(), MDB_CP_COMPACT);

                // create a temporary db
                auto temp = openEnv(compactDir);

                // copy data
                compactDatabase(env_, temp);

                // close envs
                temp.close();
                env_.close();

                // swap the databases and delete old one
                QDir().rename(cacheDirectory_, toDeleteDir);
                QDir().rename(compactDir, cacheDirectory_);
                QDir(toDeleteDir).removeRecursively();

                // reopen env
                env_ = openEnv(cacheDirectory_);
            }
        }
    } catch (const lmdb::error &e) {
        if (e.code() != MDB_VERSION_MISMATCH && e.code() != MDB_INVALID) {
            throw std::runtime_error("LMDB initialization failed" + std::string(e.what()));
        }

        nhlog::db()->warn("resetting cache due to LMDB version mismatch: {}", e.what());

        QDir stateDir(cacheDirectory_);

        auto eList = stateDir.entryList(QDir::NoDotAndDotDot);
        for (const auto &file : std::as_const(eList)) {
            if (!stateDir.remove(file))
                throw std::runtime_error(("Unable to delete file " + file).toStdString().c_str());
        }
        env_ = openEnv(cacheDirectory_);
    }

    auto txn          = lmdb::txn::begin(env_);
    syncStateDb_      = lmdb::dbi::open(txn, SYNC_STATE_DB, MDB_CREATE);
    roomsDb_          = lmdb::dbi::open(txn, ROOMS_DB, MDB_CREATE);
    spacesChildrenDb_ = lmdb::dbi::open(txn, SPACES_CHILDREN_DB, MDB_CREATE | MDB_DUPSORT);
    spacesParentsDb_  = lmdb::dbi::open(txn, SPACES_PARENTS_DB, MDB_CREATE | MDB_DUPSORT);
    invitesDb_        = lmdb::dbi::open(txn, INVITES_DB, MDB_CREATE);
    readReceiptsDb_   = lmdb::dbi::open(txn, READ_RECEIPTS_DB, MDB_CREATE);
    notificationsDb_  = lmdb::dbi::open(txn, NOTIFICATIONS_DB, MDB_CREATE);
    presenceDb_       = lmdb::dbi::open(txn, PRESENCE_DB, MDB_CREATE);

    // Device management
    devicesDb_    = lmdb::dbi::open(txn, DEVICES_DB, MDB_CREATE);
    deviceKeysDb_ = lmdb::dbi::open(txn, DEVICE_KEYS_DB, MDB_CREATE);

    // Session management
    inboundMegolmSessionDb_  = lmdb::dbi::open(txn, INBOUND_MEGOLM_SESSIONS_DB, MDB_CREATE);
    outboundMegolmSessionDb_ = lmdb::dbi::open(txn, OUTBOUND_MEGOLM_SESSIONS_DB, MDB_CREATE);
    megolmSessionDataDb_     = lmdb::dbi::open(txn, MEGOLM_SESSIONS_DATA_DB, MDB_CREATE);

    olmSessionDb_ = lmdb::dbi::open(txn, OLM_SESSIONS_DB, MDB_CREATE);

    // What rooms are encrypted
    encryptedRooms_   = lmdb::dbi::open(txn, ENCRYPTED_ROOMS_DB, MDB_CREATE);
    eventExpiryBgJob_ = lmdb::dbi::open(txn, EVENT_EXPIRATION_BG_JOB_DB, MDB_CREATE);

    [[maybe_unused]] auto verificationDb = getVerificationDb(txn);
    [[maybe_unused]] auto userKeysDb     = getUserKeysDb(txn);

    txn.commit();

    loadSecretsFromStore(
      {
        {"pickle_secret", true},
      },
      [this](const std::string &, bool, const std::string &value) { this->pickle_secret_ = value; },
      true);
}

static void
fatalSecretError()
{
    QMessageBox::critical(
      nullptr,
      QCoreApplication::translate("SecretStorage", "Failed to connect to secret storage"),
      QCoreApplication::translate(
        "SecretStorage",
        "Nheko could not connect to the secure storage to save encryption secrets to. This can "
        "have multiple reasons. Check if your D-Bus service is running and you have configured a "
        "service like KWallet, Gnome Keyring, KeePassXC or the equivalent for your platform. If "
        "you are having trouble, feel free to open an issue here: "
        "https://github.com/Nheko-Reborn/nheko/issues"),
      QMessageBox::StandardButton::Close);

    QCoreApplication::exit(1);
    exit(1);
}

static QString
secretName(std::string_view name, bool internal)
{
    auto settings = UserSettings::instance();
    return (internal ? "nheko." : "matrix.") +
           QString(
             QCryptographicHash::hash(settings->profile().toUtf8(), QCryptographicHash::Sha256)
               .toBase64()) +
           "." + QString::fromUtf8(name);
}

void
Cache::loadSecretsFromStore(
  std::vector<std::pair<std::string, bool>> toLoad,
  std::function<void(const std::string &name, bool internal, const std::string &value)> callback,
  bool databaseReadyOnFinished)
{
    auto settings = UserSettings::instance()->qsettings();

    if (toLoad.empty()) {
        this->databaseReady_ = true;

        // HACK(Nico): Some migrations would loop infinitely otherwise.
        // So we set the database to be ready, but not emit the signal, because that would start the
        // migrations again. :D
        if (databaseReadyOnFinished) {
            emit databaseReady();
            nhlog::db()->debug("Database ready");
        }
        return;
    }

    if (settings->value(QStringLiteral("run_without_secure_secrets_service"), false).toBool()) {
        for (auto &[name_, internal] : toLoad) {
            auto name  = secretName(name_, internal);
            auto value = settings->value("secrets/" + name).toString();
            if (value.isEmpty()) {
                nhlog::db()->info("Restored empty secret '{}'.", name.toStdString());
            } else {
                callback(name_, internal, value.toStdString());
            }
        }
        // if we emit the DatabaseReady signal directly it won't be received
        QTimer::singleShot(0, this, [this, callback, databaseReadyOnFinished] {
            loadSecretsFromStore({}, callback, databaseReadyOnFinished);
        });
        return;
    }

    auto [name_, internal] = toLoad.front();

    auto job = new QKeychain::ReadPasswordJob(QCoreApplication::applicationName());
    job->setAutoDelete(true);
    job->setInsecureFallback(true);
    job->setSettings(settings);
    auto name = secretName(name_, internal);
    job->setKey(name);

    connect(job,
            &QKeychain::ReadPasswordJob::finished,
            this,
            [this,
             name,
             toLoad,
             job,
             name__    = name_,
             internal_ = internal,
             callback,
             databaseReadyOnFinished](QKeychain::Job *) mutable {
                nhlog::db()->debug("Finished reading '{}'", toLoad.begin()->first);
                const QString secret = job->textData();
                if (job->error() && job->error() != QKeychain::Error::EntryNotFound) {
                    nhlog::db()->error("Restoring secret '{}' failed ({}): {}",
                                       name.toStdString(),
                                       static_cast<int>(job->error()),
                                       job->errorString().toStdString());

                    fatalSecretError();
                }
                if (secret.isEmpty()) {
                    nhlog::db()->debug("Restored empty secret '{}'.", name.toStdString());
                } else {
                    callback(name__, internal_, secret.toStdString());
                }

                // load next secret
                toLoad.erase(toLoad.begin());

                // You can't start a job from the finish signal of a job.
                QTimer::singleShot(0, this, [this, toLoad, callback, databaseReadyOnFinished] {
                    loadSecretsFromStore(toLoad, callback, databaseReadyOnFinished);
                });
            });
    nhlog::db()->debug("Reading '{}'", name_);
    job->start();
}

std::optional<std::string>
Cache::secret(std::string_view name_, bool internal)
{
    auto name = secretName(name_, internal);

    auto txn = ro_txn(env_);
    std::string_view value;
    auto db_name = "secret." + name.toStdString();
    if (!syncStateDb_.get(txn, db_name, value))
        return std::nullopt;

    mtx::secret_storage::AesHmacSha2EncryptedData data = nlohmann::json::parse(value);

    auto decrypted = mtx::crypto::decrypt(data, mtx::crypto::to_binary_buf(pickle_secret_), name_);
    if (decrypted.empty())
        return std::nullopt;
    else
        return decrypted;
}

void
Cache::storeSecret(std::string_view name_, const std::string &secret, bool internal)
{
    auto name = secretName(name_, internal);

    auto txn = lmdb::txn::begin(env_);

    auto encrypted =
      mtx::crypto::encrypt(secret, mtx::crypto::to_binary_buf(pickle_secret_), name_);

    auto db_name = "secret." + name.toStdString();
    syncStateDb_.put(txn, db_name, nlohmann::json(encrypted).dump());
    txn.commit();
    emit secretChanged(std::string(name_));
}

void
Cache::deleteSecret(std::string_view name_, bool internal)
{
    auto name = secretName(name_, internal);

    auto txn = lmdb::txn::begin(env_);
    std::string_view value;
    auto db_name = "secret." + name.toStdString();
    syncStateDb_.del(txn, db_name, value);
    txn.commit();
}

void
Cache::storeSecretInStore(const std::string name_, const std::string secret)
{
    auto name = secretName(name_, true);

    auto settings = UserSettings::instance()->qsettings();
    if (settings->value(QStringLiteral("run_without_secure_secrets_service"), false).toBool()) {
        settings->setValue("secrets/" + name, QString::fromStdString(secret));
        // if we emit the signal directly it won't be received
        QTimer::singleShot(0, this, [this, name_] { emit secretChanged(name_); });
        nhlog::db()->info("Storing secret '{}' successful", name_);
        return;
    }

    auto job = new QKeychain::WritePasswordJob(QCoreApplication::applicationName());
    job->setAutoDelete(true);
    job->setInsecureFallback(true);
    job->setSettings(settings);

    job->setKey(name);

    job->setTextData(QString::fromStdString(secret));

    QObject::connect(
      job,
      &QKeychain::WritePasswordJob::finished,
      this,
      [name_, this](QKeychain::Job *job) {
          if (job->error()) {
              nhlog::db()->warn(
                "Storing secret '{}' failed: {}", name_, job->errorString().toStdString());
              fatalSecretError();
          } else {
              // if we emit the signal directly, qtkeychain breaks and won't execute new
              // jobs. You can't start a job from the finish signal of a job.
              QTimer::singleShot(0, this, [this, name_] { emit secretChanged(name_); });
              nhlog::db()->info("Storing secret '{}' successful", name_);
          }
      },
      Qt::ConnectionType::DirectConnection);
    job->start();
}

void
Cache::deleteSecretFromStore(const std::string name, bool internal)
{
    auto name_ = secretName(name, internal);

    auto settings = UserSettings::instance()->qsettings();
    if (settings->value(QStringLiteral("run_without_secure_secrets_service"), false).toBool()) {
        settings->remove("secrets/" + name_);
        // if we emit the signal directly it won't be received
        QTimer::singleShot(0, this, [this, name] { emit secretChanged(name); });
        return;
    }

    auto job = new QKeychain::DeletePasswordJob(QCoreApplication::applicationName());
    job->setAutoDelete(true);
    job->setInsecureFallback(true);
    job->setSettings(settings);

    job->setKey(name_);

    job->connect(
      job, &QKeychain::Job::finished, this, [this, name]() { emit secretChanged(name); });
    job->start();
}

std::string
Cache::pickleSecret()
{
    if (pickle_secret_.empty()) {
        this->pickle_secret_ = mtx::client::utils::random_token(64, true);
        storeSecretInStore("pickle_secret", pickle_secret_);
    }

    return pickle_secret_;
}

void
Cache::storeEventExpirationProgress(const std::string &room,
                                    const std::string &expirationSettings,
                                    const std::string &stopMarker)
{
    nlohmann::json j;
    j["s"] = expirationSettings;
    j["m"] = stopMarker;

    auto txn = lmdb::txn::begin(env_);
    eventExpiryBgJob_.put(txn, room, j.dump());
    txn.commit();
}

std::string
Cache::loadEventExpirationProgress(const std::string &room, const std::string &expirationSettings)

{
    try {
        auto txn = ro_txn(env_);
        std::string_view data;
        if (!eventExpiryBgJob_.get(txn, room, data))
            return "";

        auto j = nlohmann::json::parse(data);
        if (j.value("s", "") == expirationSettings)
            return j.value("m", "");
    } catch (...) {
        return "";
    }
    return "";
}

void
Cache::setEncryptedRoom(lmdb::txn &txn, const std::string &room_id)
{
    nhlog::db()->info("mark room {} as encrypted", room_id);

    encryptedRooms_.put(txn, room_id, "0");
}

bool
Cache::isRoomEncrypted(const std::string &room_id)
{
    std::string_view unused;

    auto txn = ro_txn(env_);
    auto res = encryptedRooms_.get(txn, room_id, unused);

    return res;
}

std::optional<mtx::events::state::Encryption>
Cache::roomEncryptionSettings(const std::string &room_id)
{
    using namespace mtx::events;
    using namespace mtx::events::state;

    try {
        auto txn      = ro_txn(env_);
        auto statesdb = getStatesDb(txn, room_id);
        std::string_view event;
        bool res = statesdb.get(txn, to_string(mtx::events::EventType::RoomEncryption), event);

        if (res) {
            try {
                StateEvent<Encryption> msg =
                  nlohmann::json::parse(event).get<StateEvent<Encryption>>();

                return msg.content;
            } catch (const nlohmann::json::exception &e) {
                nhlog::db()->warn("failed to parse m.room.encryption event: {}", e.what());
                return Encryption{};
            }
        }
    } catch (lmdb::error &) {
    }

    return std::nullopt;
}

mtx::crypto::ExportedSessionKeys
Cache::exportSessionKeys()
{
    using namespace mtx::crypto;

    ExportedSessionKeys keys;

    auto txn    = ro_txn(env_);
    auto cursor = lmdb::cursor::open(txn, inboundMegolmSessionDb_);

    std::string_view key, value;
    while (cursor.get(key, value, MDB_NEXT)) {
        ExportedSession exported;
        MegolmSessionIndex index;

        auto saved_session = unpickle<InboundSessionObject>(std::string(value), pickle_secret_);

        try {
            index = nlohmann::json::parse(key).get<MegolmSessionIndex>();
        } catch (const nlohmann::json::exception &e) {
            nhlog::db()->critical("failed to export megolm session: {}", e.what());
            continue;
        }

        try {
            using namespace mtx::crypto;

            std::string_view v;
            if (megolmSessionDataDb_.get(txn, nlohmann::json(index).dump(), v)) {
                auto data           = nlohmann::json::parse(v).get<GroupSessionData>();
                exported.sender_key = data.sender_key;
                if (!data.sender_claimed_ed25519_key.empty())
                    exported.sender_claimed_keys["ed25519"] = data.sender_claimed_ed25519_key;
                exported.forwarding_curve25519_key_chain = data.forwarding_curve25519_key_chain;
            } else {
                continue;
            }

        } catch (std::exception &e) {
            nhlog::db()->error("Failed to retrieve Megolm Session Data: {}", e.what());
            continue;
        }

        exported.room_id     = index.room_id;
        exported.session_id  = index.session_id;
        exported.session_key = export_session(saved_session.get(), -1);

        keys.sessions.push_back(exported);
    }

    cursor.close();

    return keys;
}

void
Cache::importSessionKeys(const mtx::crypto::ExportedSessionKeys &keys)
{
    std::size_t importCount = 0;

    auto txn = lmdb::txn::begin(env_);
    for (const auto &s : keys.sessions) {
        MegolmSessionIndex index;
        index.room_id    = s.room_id;
        index.session_id = s.session_id;

        GroupSessionData data{};
        data.sender_key                      = s.sender_key;
        data.forwarding_curve25519_key_chain = s.forwarding_curve25519_key_chain;
        data.trusted                         = false;

        if (s.sender_claimed_keys.count("ed25519"))
            data.sender_claimed_ed25519_key = s.sender_claimed_keys.at("ed25519");

        try {
            auto exported_session = mtx::crypto::import_session(s.session_key);

            using namespace mtx::crypto;
            const auto key = nlohmann::json(index).dump();
            const auto pickled =
              pickle<InboundSessionObject>(exported_session.get(), pickle_secret_);

            std::string_view value;
            if (inboundMegolmSessionDb_.get(txn, key, value)) {
                auto oldSession =
                  unpickle<InboundSessionObject>(std::string(value), pickle_secret_);
                if (olm_inbound_group_session_first_known_index(exported_session.get()) >=
                    olm_inbound_group_session_first_known_index(oldSession.get())) {
                    nhlog::crypto()->warn(
                      "Not storing inbound session with newer or equal first known index");
                    continue;
                }
            }

            inboundMegolmSessionDb_.put(txn, key, pickled);
            megolmSessionDataDb_.put(txn, key, nlohmann::json(data).dump());

            ChatPage::instance()->receivedSessionKey(index.room_id, index.session_id);
            importCount++;
        } catch (const mtx::crypto::olm_exception &e) {
            nhlog::crypto()->critical(
              "failed to import inbound megolm session {}: {}", index.session_id, e.what());
            continue;
        } catch (const lmdb::error &e) {
            nhlog::crypto()->critical(
              "failed to save inbound megolm session {}: {}", index.session_id, e.what());
            continue;
        }
    }
    txn.commit();

    nhlog::crypto()->info("Imported {} out of {} keys", importCount, keys.sessions.size());
}

//
// Session Management
//

void
Cache::saveInboundMegolmSession(const MegolmSessionIndex &index,
                                mtx::crypto::InboundGroupSessionPtr session,
                                const GroupSessionData &data)
{
    using namespace mtx::crypto;
    const auto key     = nlohmann::json(index).dump();
    const auto pickled = pickle<InboundSessionObject>(session.get(), pickle_secret_);

    auto txn = lmdb::txn::begin(env_);

    std::string_view value;
    if (inboundMegolmSessionDb_.get(txn, key, value)) {
        auto oldSession = unpickle<InboundSessionObject>(std::string(value), pickle_secret_);

        auto newIndex = olm_inbound_group_session_first_known_index(session.get());
        auto oldIndex = olm_inbound_group_session_first_known_index(oldSession.get());

        // merge trusted > untrusted
        // first known index minimum
        if (megolmSessionDataDb_.get(txn, key, value)) {
            auto oldData = nlohmann::json::parse(value).get<GroupSessionData>();
            if (oldData.trusted && newIndex >= oldIndex) {
                nhlog::crypto()->warn(
                  "Not storing inbound session of lesser trust or bigger index.");
                return;
            }

            oldData.trusted = data.trusted || oldData.trusted;

            if (newIndex < oldIndex) {
                inboundMegolmSessionDb_.put(txn, key, pickled);
                oldData.message_index = newIndex;
            }

            megolmSessionDataDb_.put(txn, key, nlohmann::json(oldData).dump());
            txn.commit();
            return;
        }
    }

    inboundMegolmSessionDb_.put(txn, key, pickled);
    megolmSessionDataDb_.put(txn, key, nlohmann::json(data).dump());
    txn.commit();
}

mtx::crypto::InboundGroupSessionPtr
Cache::getInboundMegolmSession(const MegolmSessionIndex &index)
{
    using namespace mtx::crypto;

    try {
        auto txn        = ro_txn(env_);
        std::string key = nlohmann::json(index).dump();
        std::string_view value;

        if (inboundMegolmSessionDb_.get(txn, key, value)) {
            auto session = unpickle<InboundSessionObject>(std::string(value), pickle_secret_);
            return session;
        }
    } catch (std::exception &e) {
        nhlog::db()->error("Failed to get inbound megolm session {}", e.what());
    }

    return nullptr;
}