diff --git a/app/graphicsview.cpp b/app/graphicsview.cpp
index 1573014..577c907 100644
--- a/app/graphicsview.cpp
+++ b/app/graphicsview.cpp
@@ -263,9 +263,13 @@ void GraphicsView::mouseMoveEvent(QMouseEvent *event)
 
 void GraphicsView::mouseReleaseEvent(QMouseEvent *event)
 {
-    QGraphicsItem *item = itemAt(event->pos());
-    if (!item) {
+    if (event->button() == Qt::ForwardButton || event->button() == Qt::BackButton) {
         event->ignore();
+    } else {
+        QGraphicsItem *item = itemAt(event->pos());
+        if (!item) {
+            event->ignore();
+        }
     }
 
     return QGraphicsView::mouseReleaseEvent(event);
diff --git a/app/mainwindow.cpp b/app/mainwindow.cpp
index e5e87fb..3575679 100644
--- a/app/mainwindow.cpp
+++ b/app/mainwindow.cpp
@@ -272,15 +272,6 @@ void MainWindow::mousePressEvent(QMouseEvent *event)
         event->accept();
     }
 
-    // It seems the forward/back mouse button won't generate a key event [1] so we can't use
-    // QShortcut or QKeySequence to indicate these shortcuts, so we do it here.
-    // Reference:
-    // [1]: https://codereview.qt-project.org/c/qt/qtbase/+/177475
-    if (event->buttons() & Qt::ForwardButton || event->buttons() & Qt::BackButton) {
-        event->buttons() & Qt::BackButton ? galleryNext() : galleryPrev();
-        event->accept();
-    }
-
     return FramelessWindow::mousePressEvent(event);
 }
 
@@ -304,6 +295,15 @@ void MainWindow::mouseReleaseEvent(QMouseEvent *event)
 {
     m_clickedOnWindow = false;
 
+    // It seems the forward/back mouse button won't generate a key event [1] so we can't use
+    // QShortcut or QKeySequence to indicate these shortcuts, so we do it here.
+    // Reference:
+    // [1]: https://codereview.qt-project.org/c/qt/qtbase/+/177475
+    if (event->button() == Qt::ForwardButton || event->button() == Qt::BackButton) {
+        event->button() == Qt::BackButton ? galleryNext() : galleryPrev();
+        event->accept();
+    }
+
     return FramelessWindow::mouseReleaseEvent(event);
 }