diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 4ca5b2551e1288c3b5c193d287881f8a832917a3..12893e7467090fecaccbfd7e1ddaeb8e69721589 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -739,9 +739,18 @@ static int vhost_user_get_vring_base(struct vhost_dev *dev,
         .payload.state = *ring,
         .hdr.size = sizeof(msg.payload.state),
     };
+    struct vhost_user *u = dev->opaque;
+    VhostUserState *user = u->user;
 
     vhost_user_host_notifier_remove(dev, ring->index);
 
+    /* For server mode, free mmap notifier from client */
+    if (user->notifier[ring->index].addr) {
+        object_unparent(OBJECT(&user->notifier[ring->index].mr));
+        munmap(user->notifier[ring->index].addr, qemu_real_host_page_size);
+        user->notifier[ring->index].addr = NULL;
+    }
+
     if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
         return -1;
     }
diff --git a/net/vhost-user.c b/net/vhost-user.c
index 014199d6008f380ef0f296655339356b4753a6d3..5c07ae18bca04b521a6ec50fec7e199ab74d8014 100644
--- a/net/vhost-user.c
+++ b/net/vhost-user.c
@@ -226,23 +226,28 @@ static void chr_closed_bh(void *opaque)
     NetClientState *ncs[MAX_QUEUE_NUM];
     NetVhostUserState *s;
     Error *err = NULL;
-    int queues;
+    int queues = 0;
+    int i = 0;
 
     queues = qemu_find_net_clients_except(name, ncs,
                                           NET_CLIENT_DRIVER_NIC,
                                           MAX_QUEUE_NUM);
     assert(queues < MAX_QUEUE_NUM);
 
-    s = DO_UPCAST(NetVhostUserState, nc, ncs[0]);
-
-    if (s->vhost_net) {
-        s->acked_features = vhost_net_get_acked_features(s->vhost_net);
+    for (i = 0; i < queues; i++) {
+        s = DO_UPCAST(NetVhostUserState, nc, ncs[i]);
+        if (s->vhost_net) {
+            s->acked_features = vhost_net_get_acked_features(s->vhost_net);
+        }
     }
 
     qmp_set_link(name, false, &err);
 
-    qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, net_vhost_user_event,
-                             NULL, opaque, NULL, true);
+    for (i = 0; i < queues; i++) {
+        s = DO_UPCAST(NetVhostUserState, nc, ncs[i]);
+        qemu_chr_fe_set_handlers(&s->chr, NULL, NULL, net_vhost_user_event,
+                                 NULL, opaque, NULL, true);
+    }
 
     if (err) {
         error_report_err(err);