repmgr -f /etc/repmgr.conf witness register -h node1

 primary_visibility_consensus=true

failover_validation_command=/path/to/script.sh %n

repmgr cluster event --event=xxxxx (事件类型)

shared_preload_libraries = 'repmgr'

•failover =automatic•promote_command =/'usr/bin/repmgr standby promote -f /etc/repmgr.conf --log-to-file'•follow_command= '/usr/bin/repmgr standby follow -f /etc/repmgr.conf --log-to-file --upstream-node-id=%n'

priorityfailover_validation_commandprimary_visibility_consensusalways_promotestandby_disconnect_on_failoverelection_rerun_intervalsibling_nodes_disconnect_timeout

/var/log/repmgr/repmgrd.log {        missingok        compress        rotate 52        maxsize 100M        weekly        create 0600 postgres postgres        postrotate            /usr/bin/killall -HUP repmgrd        endscript    }

# 维护模式repmgr -f /etc/repmgr.conf service pause# 解除维护模式repmgr -f /etc/repmgr.conf service unpause# 查看当前模式repmgr -f /etc/repmgr.conf service status

/* * Failover decision for nodes attached to the current primary. * * NB: this function sets "sibling_nodes"; caller (do_primary_failover) * expects to be able to read this list */# 选举流程 Postgresql 支持级联复制。新的主节点只能在一级子节点中产生。#  sibling_nodes 兄弟节点static ElectionResultdo_election(NodeInfoList *sibling_nodes, int *new_primary_id){  int      electoral_term = -1;    #  NodeInfoListCell *cell = NULL;#   t_node_info *candidate_node = NULL;# 选举状态  election_stats stats;
  ReplInfo  local_replication_info;
  /* To collate details of nodes with primary visible for logging purposes */  PQExpBufferData nodes_with_primary_visible;
  /*   * Check if at least one server in the primary's location is visible; if   * not we'll assume a network split between this node and the primary   * location, and not promote any standby.   *   * NOTE: this function is only ever called by standbys attached to the   * current (unreachable) primary, so "upstream_node_info" will always   * contain the primary node record.   */  # 根据是否存在与故障节点位置相同的可见节点。如果都为不可见状态，则认为是网络问题。PS: 每个节点可配置location 参数指定所在的物理位置。如多数据中心场景  bool    primary_location_seen = false;
  int      nodes_with_primary_still_visible = 0;
  if (config_file_options.failover_delay > 0)  {    log_debug("sleeping %i seconds (\"failover_delay\") before initiating failover",          config_file_options.failover_delay);    sleep(config_file_options.failover_delay);  }
  /* we're visible */   #可见节点数  stats.visible_nodes = 1;  #与当前node为相同上游的节点数 shared_upstream_nodes = sibling_nodes + 1(当前节点)  stats.shared_upstream_nodes = 0;  # 集群中所有节点数  stats.all_nodes = 0;    # SELECT term FROM repmgr.voting_term; 初始值为1 , 执行repmgr standby promote加一  electoral_term = get_current_term(local_conn);
  if (electoral_term == -1)  {    log_error(_("unable to determine electoral term"));
    return ELECTION_NOT_CANDIDATE;  }
  log_debug("do_election(): electoral term is %i", electoral_term);   # 如果repmgr.conf中failover参数设置为manual ,该节点不参与选举  if (config_file_options.failover == FAILOVER_MANUAL)  {    log_notice(_("this node is not configured for automatic failover so will not be considered as promotion candidate, and will not follow the new primary"));    log_detail(_("\"failover\" is set to \"manual\" in repmgr.conf"));    log_hint(_("manually execute \"repmgr standby follow\" to have this node follow the new primary"));
    return ELECTION_NOT_CANDIDATE;  }#如果设置priority为0 ，该节点不参与选举 。见证节点的priority=0。及见证节点到这里已经出局  /* node priority is set to zero - don't become a candidate, and lose by default */  if (local_node_info.priority <= 0)  {    log_notice(_("this node's priority is %i so will not be considered as an automatic promotion candidate"),           local_node_info.priority);
    return ELECTION_LOST;  }    #该方法 SQL 查询 视图repmgr.nodes ,获取兄弟节点。何为兄弟，与当前节点上游节点相同的节点，不包括自身。    # 注意一点。见证节点的也算在兄弟节点中。因为在nodes表中他们的upstream_node_id 相同     # SELECT REPMGR_NODES_COLUMNS FROM repmgr.nodes n  WHERE n.upstream_node_id = %i  AND n.node_id != %i    AND n.active IS TRUE ORDER BY n.node_id    /* get all active nodes attached to upstream, excluding self */  get_active_sibling_node_records(local_conn,                  local_node_info.node_id,                  upstream_node_info.node_id,                  sibling_nodes);
  log_info(_("%i active sibling nodes registered"), sibling_nodes->node_count);
  stats.shared_upstream_nodes = sibling_nodes->node_count + 1;    #获取集群中所有节点数量 SELECT count(*) FROM repmgr.nodes n   get_all_nodes_count(local_conn, &stats.all_nodes);
  log_info(_("%i total nodes registered"), stats.all_nodes);    #判断记录当前节点位置是否与 上游节点（主节点）位置相同  if (strncmp(upstream_node_info.location, local_node_info.location, MAXLEN) != 0)  {    log_info(_("primary node \"%s\" (ID: %i) has location \"%s\", this node's location is \"%s\""),         upstream_node_info.node_name,         upstream_node_info.node_id,         upstream_node_info.location,         local_node_info.location);  }  else   {    log_info(_("primary node  \"%s\" (ID: %i) and this node have the same location (\"%s\")"),         upstream_node_info.node_name,         upstream_node_info.node_id,         local_node_info.location);  }
  local_node_info.last_wal_receive_lsn = InvalidXLogRecPtr;  # 没有兄弟节点，没有见证节点，独生子。如果与主节点位置相同直接继承王位，江湖中的打打杀杀在这里不存在。  /* fast path if no other standbys (or witness) exists - normally win by default */  if (sibling_nodes->node_count == 0)  {  # 比较上游节点（原主节点）与当前节点的位置，如果相同    if (strncmp(upstream_node_info.location, local_node_info.location, MAXLEN) == 0)    {# 升级为主服务前的配置的验证脚本，是否返回值为0      if (config_file_options.failover_validation_command[0] != '\0')      {         return execute_failover_validation_command(&local_node_info, &stats);      }
      log_info(_("no other sibling nodes - we win by default"));          # 胜出，新王诞生。      return ELECTION_WON;    }    else    {      /*       * If primary and standby have different locations set, the assumption       * is that no action should be taken as we can't tell whether there's       * been a network interruption or not.       *       * Normally a situation with primary and standby in different physical       * locations would be handled by leaving the location as "default" and       * setting up a witness server in the primary's location.       */      log_debug("no other nodes, but primary and standby locations differ");           # 虽然只有一个从节点，但是与主节点的位置不同，不能进行升级。集群进行降级处理。很遗憾。           # 相当于 主节点相同位置的节点都不可见。被认为网络分区故障。      monitoring_state = MS_DEGRADED;      INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
      return ELECTION_NOT_CANDIDATE;    }  }  # 当有多个兄弟的时候  else   {    /* standby nodes found - check if we're in the primary location before checking theirs */ # 当前节点是否与主节点位置相同    if (strncmp(upstream_node_info.location, local_node_info.location, MAXLEN) == 0)    {   # 存在与主节点位置相同的直接从节点      primary_location_seen = true;    }  }
  /* get our lsn */ #   if (get_replication_info(local_conn, STANDBY, &local_replication_info) == false)  {    log_error(_("unable to retrieve replication information for local node"));    return ELECTION_LOST;  }     # wal 回放状态为 paused ，尝试恢复wal回放状态 , resume失败 当前节点不参与选举  /* check if WAL replay on local node is paused */  if (local_replication_info.wal_replay_paused == true)  {    log_debug("WAL replay is paused");    if (local_replication_info.last_wal_receive_lsn > local_replication_info.last_wal_replay_lsn)    {      log_warning(_("WAL replay on this node is paused and WAL is pending replay"));      log_detail(_("replay paused at %X/%X; last WAL received is %X/%X"),             format_lsn(local_replication_info.last_wal_replay_lsn),             format_lsn(local_replication_info.last_wal_receive_lsn));    }    # 尝试恢复wal状态    /* attempt to resume WAL replay - unlikely this will fail, but just in case */    if (resume_wal_replay(local_conn) == false)    {      log_error(_("unable to resume WAL replay"));      log_detail(_("this node cannot be reliably promoted"));      # resume 失败，该节点不参与选举      return ELECTION_LOST;    }
    log_notice(_("WAL replay forcibly resumed"));  }
  local_node_info.last_wal_receive_lsn = local_replication_info.last_wal_receive_lsn;
  log_info(_("local node's last receive lsn: %X/%X"), format_lsn(local_node_info.last_wal_receive_lsn));
  /* pointer to "winning" node, initially self */  candidate_node = &local_node_info;
  initPQExpBuffer(&nodes_with_primary_visible);   # 众候选节点进入选举会场，开始厮杀。  for (cell = sibling_nodes->head; cell; cell = cell->next)  {    ReplInfo  sibling_replication_info;
    log_info(_("checking state of sibling node \"%s\" (ID: %i)"),         cell->node_info->node_name,         cell->node_info->node_id);
    /* assume the worst case */    cell->node_info->node_status = NODE_STATUS_UNKNOWN;
    cell->node_info->conn = establish_db_connection(cell->node_info->conninfo, false);
    if (PQstatus(cell->node_info->conn) != CONNECTION_OK)    {      close_connection(&cell->node_info->conn);
      continue;    }
    cell->node_info->node_status = NODE_STATUS_UP;
    stats.visible_nodes++;
    /*     * see if the node is in the primary's location (but skip the check if     * we've seen a node there already)     */    if (primary_location_seen == false)    {      if (strncmp(cell->node_info->location, upstream_node_info.location, MAXLEN) == 0)      {        log_debug("node %i in primary location \"%s\"",              cell->node_info->node_id,              cell->node_info->location);        primary_location_seen = true;      }    }
    /*     * check if repmgrd running - skip if not     *     * TODO: include pid query in replication info query?     *     * NOTE: from Pg12 we could execute "pg_promote()" from a running repmgrd;     * here we'll need to find a way of ensuring only one repmgrd does this     */    if (repmgrd_get_pid(cell->node_info->conn) == UNKNOWN_PID)    {      log_warning(_("repmgrd not running on node \"%s\" (ID: %i), skipping"),            cell->node_info->node_name,            cell->node_info->node_id);      continue;    }
    if (get_replication_info(cell->node_info->conn, cell->node_info->type, &sibling_replication_info) == false)    {      log_warning(_("unable to retrieve replication information for node \"%s\" (ID: %i), skipping"),            cell->node_info->node_name,            cell->node_info->node_id);      continue;    }
    /*     * Check if node is not in recovery - it may have been promoted     * outside of the failover mechanism, in which case we may be able     * to follow it.     */
    # 有兄弟节点被手动提升为主节点    if (sibling_replication_info.in_recovery == false && cell->node_info->type != WITNESS)    {      bool can_follow;
      log_warning(_("node \"%s\" (ID: %i) is not in recovery"),            cell->node_info->node_name,            cell->node_info->node_id);
      /*       * Node is not in recovery, but still reporting an upstream       * node ID; possible it was promoted manually (e.g. with "pg_ctl promote"),       * or (less likely) the node's repmgrd has just switched to primary       * monitoring node but has not yet unset the upstream node ID in       * shared memory. Either way, log this.       */      if (sibling_replication_info.upstream_node_id != UNKNOWN_NODE_ID)      {        log_warning(_("node \"%s\" (ID: %i) still reports its upstream is node %i, last seen %i second(s) ago"),              cell->node_info->node_name,              cell->node_info->node_id,              sibling_replication_info.upstream_node_id,              sibling_replication_info.upstream_last_seen);      }      #检验当前节点是否能follow 被提升的兄弟节点      can_follow = check_node_can_follow(local_conn,                         local_node_info.last_wal_receive_lsn,                         cell->node_info->conn,                         cell->node_info);      # 在选举结果前有节点手动提升节点为主节点。并且当前节点可以follower。当前节点不在参与选举      if (can_follow == true)      {        *new_primary_id = cell->node_info->node_id;        termPQExpBuffer(&nodes_with_primary_visible);        return ELECTION_CANCELLED;      }      # 如果不能follower主节点。这种情况会很棘手。      /*       * Tricky situation here - we'll assume the node is a rogue primary       */      log_warning(_("not possible to attach to node \"%s\" (ID: %i), ignoring"),            cell->node_info->node_name,            cell->node_info->node_id);      continue;    }    else    {      log_info(_("node \"%s\" (ID: %i) reports its upstream is node %i, last seen %i second(s) ago"),           cell->node_info->node_name,           cell->node_info->node_id,           sibling_replication_info.upstream_node_id,           sibling_replication_info.upstream_last_seen);    }
    /* check if WAL replay on node is paused */    if (sibling_replication_info.wal_replay_paused == true)    {      /*       * Theoretically the repmgrd on the node should have resumed WAL play       * at this point.       */      if (sibling_replication_info.last_wal_receive_lsn > sibling_replication_info.last_wal_replay_lsn)      {        log_warning(_("WAL replay on node \"%s\" (ID: %i) is paused and WAL is pending replay"),              cell->node_info->node_name,              cell->node_info->node_id);      }    }
    /*     * Check if node has seen primary "recently" - if so, we may have "partial primary visibility".     * For now we'll assume the primary is visible if it's been seen less than     * monitor_interval_secs * 2 seconds ago. We may need to adjust this, and/or make the value     * configurable.     */
    if (sibling_replication_info.upstream_last_seen >= 0 && sibling_replication_info.upstream_last_seen < (config_file_options.monitor_interval_secs * 2))    {      if (sibling_replication_info.upstream_node_id != upstream_node_info.node_id)      {        log_warning(_("assumed sibling node \"%s\" (ID: %i) monitoring different upstream node %i"),              cell->node_info->node_name,              cell->node_info->node_id,              sibling_replication_info.upstream_node_id);
      }      else      {        nodes_with_primary_still_visible++;        log_notice(_("%s node \"%s\" (ID: %i) last saw primary node %i second(s) ago, considering primary still visible"),               get_node_type_string(cell->node_info->type),               cell->node_info->node_name,               cell->node_info->node_id,               sibling_replication_info.upstream_last_seen);        appendPQExpBuffer(&nodes_with_primary_visible,                  " - node \"%s\" (ID: %i): %i second(s) ago\n",                  cell->node_info->node_name,                  cell->node_info->node_id,                  sibling_replication_info.upstream_last_seen);      }    }    else    {      log_info(_("%s node \"%s\" (ID: %i) last saw primary node %i second(s) ago"),           get_node_type_string(cell->node_info->type),           cell->node_info->node_name,           cell->node_info->node_id,           sibling_replication_info.upstream_last_seen);    }
    # 见证节点不参与选举    /* don't interrogate a witness server */    if (cell->node_info->type == WITNESS)    {      log_debug("node %i is witness, not querying state", cell->node_info->node_id);      continue;    }        # priority=0 的节点不参与选举    /* don't check 0-priority nodes */    if (cell->node_info->priority <= 0)    {      log_info(_("node \"%s\" (ID: %i) has priority of %i, skipping"),             cell->node_info->node_name,             cell->node_info->node_id,             cell->node_info->priority);      continue;    }

    /* get node's last receive LSN - if "higher" than current winner, current node is candidate */    cell->node_info->last_wal_receive_lsn = sibling_replication_info.last_wal_receive_lsn;
    log_info(_("last receive LSN for sibling node \"%s\" (ID: %i) is: %X/%X"),         cell->node_info->node_name,         cell->node_info->node_id,         format_lsn(cell->node_info->last_wal_receive_lsn));
    /* compare LSN */ # 比较LSN    if (cell->node_info->last_wal_receive_lsn > candidate_node->last_wal_receive_lsn)    {      /* other node is ahead */      log_info(_("node \"%s\" (ID: %i) is ahead of current candidate \"%s\" (ID: %i)"),           cell->node_info->node_name,           cell->node_info->node_id,           candidate_node->node_name,           candidate_node->node_id);
      candidate_node = cell->node_info;    }    # LSN 相同比较 priority,priority也相同然后比较 node_id    /* LSN is same - tiebreak on priority, then node_id */    #LSN 对比    else if (cell->node_info->last_wal_receive_lsn == candidate_node->last_wal_receive_lsn)    {      log_info(_("node \"%s\" (ID: %i) has same LSN as current candidate \"%s\" (ID: %i)"),           cell->node_info->node_name,           cell->node_info->node_id,           candidate_node->node_name,           candidate_node->node_id);            #priority 对比      if (cell->node_info->priority > candidate_node->priority)      {        log_info(_("node \"%s\" (ID: %i) has higher priority (%i) than current candidate \"%s\" (ID: %i) (%i)"),             cell->node_info->node_name,             cell->node_info->node_id,             cell->node_info->priority,             candidate_node->node_name,             candidate_node->node_id,             candidate_node->priority);
        candidate_node = cell->node_info;      }      else if (cell->node_info->priority == candidate_node->priority)      {        if (cell->node_info->node_id < candidate_node->node_id)        {          log_info(_("node \"%s\" (ID: %i) has same priority but lower node_id than current candidate \"%s\" (ID: %i)"),               cell->node_info->node_name,               cell->node_info->node_id,               candidate_node->node_name,               candidate_node->node_id);
          candidate_node = cell->node_info;        }      }      else      {        log_info(_("node \"%s\" (ID: %i) has lower priority (%i) than current candidate \"%s\" (ID: %i) (%i)"),             cell->node_info->node_name,             cell->node_info->node_id,             cell->node_info->priority,             candidate_node->node_name,             candidate_node->node_id,             candidate_node->priority);      }    }  }    # 网络分区，降级。  if (primary_location_seen == false)  {    log_notice(_("no nodes from the primary location \"%s\" visible - assuming network split"),           upstream_node_info.location);    log_detail(_("node will enter degraded monitoring state waiting for reconnect"));
    monitoring_state = MS_DEGRADED;    INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
    reset_node_voting_status();
    termPQExpBuffer(&nodes_with_primary_visible);
    return ELECTION_CANCELLED;  }    # 主节点仍可见  if (nodes_with_primary_still_visible > 0)  {    log_info(_("%i nodes can see the primary"),           nodes_with_primary_still_visible);
    log_detail(_("following nodes can see the primary:\n%s"),           nodes_with_primary_visible.data);    #主节点可见性共识    if (config_file_options.primary_visibility_consensus == true)    {      log_notice(_("cancelling failover as some nodes can still see the primary"));      monitoring_state = MS_DEGRADED;      INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
      reset_node_voting_status();
      termPQExpBuffer(&nodes_with_primary_visible);
      return ELECTION_CANCELLED;    }  }
  termPQExpBuffer(&nodes_with_primary_visible);
  log_info(_("visible nodes: %i; total nodes: %i; no nodes have seen the primary within the last %i seconds"),       stats.visible_nodes,       stats.shared_upstream_nodes,       (config_file_options.monitor_interval_secs * 2));  # 同级节点中大部分不可见，不参与选举  if (stats.visible_nodes <= (stats.shared_upstream_nodes / 2.0))  {    log_notice(_("unable to reach a qualified majority of nodes"));    log_detail(_("node will enter degraded monitoring state waiting for reconnect"));
    monitoring_state = MS_DEGRADED;    INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
    reset_node_voting_status();
    return ELECTION_CANCELLED;  }
  log_notice(_("promotion candidate is \"%s\" (ID: %i)"),         candidate_node->node_name,         candidate_node->node_id);   # 新选出的节点为当前节点  if (candidate_node->node_id == local_node_info.node_id)  {    /*     * If "failover_validation_command" is set, execute that command     * and decide the result based on the command's output     */      # 验证failover_validation_command 脚本的执行    if (config_file_options.failover_validation_command[0] != '\0')    {      return execute_failover_validation_command(candidate_node, &stats);    }    # 当前节点宣布胜出，为主节点    return ELECTION_WON;  }
  return ELECTION_LOST;}if (primary_location_seen == false)  {    log_notice(_("no nodes from the primary location \"%s\" visible - assuming network split"),           upstream_node_info.location);    log_detail(_("node will enter degraded monitoring state waiting for reconnect"));
    monitoring_state = MS_DEGRADED;    INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
    reset_node_voting_status();
    termPQExpBuffer(&nodes_with_primary_visible);
    return ELECTION_CANCELLED;  }
  if (nodes_with_primary_still_visible > 0)  {    log_info(_("%i nodes can see the primary"),           nodes_with_primary_still_visible);
    log_detail(_("following nodes can see the primary:\n%s"),           nodes_with_primary_visible.data);
    if (config_file_options.primary_visibility_consensus == true)    {      log_notice(_("cancelling failover as some nodes can still see the primary"));      monitoring_state = MS_DEGRADED;      INSTR_TIME_SET_CURRENT(degraded_monitoring_start);
      reset_node_voting_status();
      termPQExpBuffer(&nodes_with_primary_visible);
      return ELECTION_CANCELLED;    }  }

ELECTION_NOT_CANDIDATELECTION_WONELECTION_LOSTELECTION_CANCELLEDELECTION_RERUN

设置参数 child_nodes_connected_min_count触发事件 child_nodes_disconnect_command 中定义回调方法将主节点降级为只读，或关闭。