同じIDを複数保存してしまう

RealmでIDを保存したいです。
手順としてはdidTapFollow関数で、同じidがあったらidを削除し、なかったらidを追加します。
しかし、３回押さないとボタンが切り替わらなかったり、その過程でidが複数保存されてしまいます。
原因がどこにあるのか検討がつかないため、下記のコードを見てもしわかる方がいらしたら対策方法を
教えてくださると幸いです。

import UIKit
import RealmSwift

enum NotificationType: Int {
    case follow
    case like
    case reply
    case retweet
    case mention
}

private let realm = try! Realm()

class NotificationViewController: UITableViewController {
    let currentUser = CurrentUser.shared
    
    private let type: NotificationType = .follow
    
    var notificationMessage: String {
        switch type {
        case .follow:
            return " Started following you"
        case .like:
            return " Liked your tweet"
        case .reply:
            return " Replied to your tweet"
        case .retweet:
            return " Retweeted your tweet"
        case .mention:
            return " Mentioned you in a tweet"
        }
    }
    
    var shouldHideFollowButton: Bool {
        return type != .follow
    }
    
    private let reuseIdentifier = "NotificationCell"
    
    private var users = Array(realm.objects(User.self))

    override func viewDidLoad() {
        super.viewDidLoad()
        configureUI()
    }
    
    override func viewWillAppear(_ animated: Bool) {
        super.viewWillAppear(animated)
        navigationController?.navigationBar.isHidden = false
        navigationController?.navigationBar.barStyle = .default
    }
    
    @objc func handleRefresh(){
        fetchNotifications()
    }
    
    func configureUI(){
        view.backgroundColor = .white
        navigationItem.title = "Notifications"
        tableView.register(NotificationCell.self, forCellReuseIdentifier: reuseIdentifier)
        tableView.rowHeight = 60
        tableView.separatorStyle = .none
        let refreshControl = UIRefreshControl()
        tableView.refreshControl = refreshControl
        refreshControl.addTarget(self, action: #selector(handleRefresh), for: .valueChanged)
    }
    
}

extension NotificationViewController {
    
    override func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int {
        return users.count
    }
    
    override func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
        let cell = tableView.dequeueReusableCell(withIdentifier: reuseIdentifier, for: indexPath) as! NotificationCell
        cell.delegate = self
        cell.selectionStyle = UITableViewCell.SelectionStyle.none
        cell.contentView.isUserInteractionEnabled = false
        let userIndex = users[indexPath.row]
        cell.profileImageView.image = UIImage(data: userIndex.profileImage!)
        cell.notificationLabel.text = userIndex.username
        let containIndex = currentUser.user.followUserList.indices.contains(userIndex.id)
        if containIndex {
            cell.followButton.setTitle("unfollow", for: .normal)
            cell.followButton.backgroundColor = .white
            cell.followButton.setTitleColor(.twitterBlue, for: .normal)
        } else {
            cell.followButton.setTitle("follow", for: .normal)
            cell.followButton.backgroundColor = .twitterBlue
            cell.followButton.setTitleColor(.white, for: .normal)
        }
        return cell
    }
    
}

extension NotificationViewController: NotificationCellDelegate {
    
    func didTapFollow(_ cell: NotificationCell) {
        let indexPath = tableView.indexPath(for: cell)
        let userIndex = users[indexPath!.row]
        try! realm.write {
            print("indexPath: (indexPath!.row)")
            print("userIndex: (userIndex.id)")
            let containIndex = currentUser.user.followUserList.indices.contains(userIndex.id)
            if containIndex == true {
                currentUser.user.followUserList.remove(at: userIndex.id)
                cell.followButton.setTitle("follow", for: .normal)
                cell.followButton.backgroundColor = .twitterBlue
                cell.followButton.setTitleColor(.white, for: .normal)
            } else {
                currentUser.user.followUserList.append(userIndex.id)
                realm.add(userIndex, update: .all)
                cell.followButton.setTitle("unfollow", for: .normal)
                cell.followButton.backgroundColor = .white
                cell.followButton.setTitleColor(.twitterBlue, for: .normal)
            }
        }
    }
    
    func didTapProfileImage(_ cell: NotificationCell) {
//        guard let user = cell.user else { return }
        let controller = ProfileController()
        navigationController?.pushViewController(controller, animated: true)
    }
    
}

class User: Object {
    
    @objc dynamic var id: Int = 0
    @objc dynamic var fullname: String = ""
    @objc dynamic var username: String = ""
    @objc dynamic var profileText: String? = ""
    @objc dynamic var profileImage: Data? = nil
    
    var tweets = List<Tweet>()
    var replyTweet = List<ReplyTweet>()

    var followUserList = List<Int>()
    
    override static func primaryKey() -> String? {
        return "id"
    }
    
}