diff --git a/include/TimelineView.h b/include/TimelineView.h
index 55f25687bf7d5a786ca90f43bf94115c55affc97..400b0db0ea1d9842d879fb48fc2e7a541f2803d7 100644
--- a/include/TimelineView.h
+++ b/include/TimelineView.h
@@ -155,6 +155,8 @@ private:
         int oldPosition_;
         int oldHeight_;
 
+        TimelineDirection lastMessageDirection_;
+
         // The events currently rendered. Used for duplicate detection.
         QMap<QString, bool> eventIds_;
         QList<PendingMessage> pending_msgs_;
diff --git a/src/TimelineView.cc b/src/TimelineView.cc
index 5463064f59172a6c53b2f9e0eb636f3098893751..2142f546db85c449d051d3a5cafdc45c517ffb98 100644
--- a/src/TimelineView.cc
+++ b/src/TimelineView.cc
@@ -96,7 +96,8 @@ TimelineView::sliderRangeChanged(int min, int max)
         if (oldPosition_ == 0 && !scroll_area_->verticalScrollBar()->isVisible())
                 newPosition = max;
 
-        scroll_area_->verticalScrollBar()->setValue(newPosition);
+        if (lastMessageDirection_ == TimelineDirection::Top)
+                scroll_area_->verticalScrollBar()->setValue(newPosition);
 }
 
 void
@@ -194,6 +195,8 @@ TimelineView::addBackwardsEvents(const QString &room_id, const RoomMessages &msg
         for (const auto &item : items)
                 addTimelineItem(item, TimelineDirection::Top);
 
+        lastMessageDirection_ = TimelineDirection::Top;
+
         QApplication::processEvents();
 
         prev_batch_token_       = msgs.end();
@@ -349,6 +352,8 @@ TimelineView::addEvents(const Timeline &timeline)
                 }
         }
 
+        lastMessageDirection_ = TimelineDirection::Bottom;
+
         QApplication::processEvents();
 
         if (isInitialSync) {
@@ -490,6 +495,8 @@ TimelineView::addUserMessage(matrix::events::MessageEventType ty, const QString
         TimelineItem *view_item = new TimelineItem(ty, user_id, body, with_sender, scroll_widget_);
         scroll_layout_->addWidget(view_item);
 
+        lastMessageDirection_ = TimelineDirection::Bottom;
+
         QApplication::processEvents();
 
         lastSender_ = user_id;
@@ -510,6 +517,8 @@ TimelineView::addUserMessage(const QString &url, const QString &filename, int tx
         TimelineItem *view_item = new TimelineItem(image, user_id, with_sender, scroll_widget_);
         scroll_layout_->addWidget(view_item);
 
+        lastMessageDirection_ = TimelineDirection::Bottom;
+
         QApplication::processEvents();
 
         lastSender_ = user_id;