Routes disappear after reloading BIRD configuration

21 May 2015

      Hi,

I've got two hosts each running BIRD within a docker container.  If I
restart BIRD on one of the hosts, I drop routes.

I have the kernel option "persist" set, running bird with the -R option and
set kernel and bgp to have graceful restart.

Restarting the service, sending in a SIGHUP or entering "configure" on
birdc all cause routes to be dropped temporarily.

Any thoughts on what might be causing this?  Is this just a configuration
issue?  I've outlined my set up below, including config files etc.

One other question, are there any plans to allow dynamic reconfiguration
(e.g. adding and removing a peer) without restarting the protocols?

Any help or advice you could give would be most welcome.

Thanks,

Rob Brockbank

Set up is as follows:

HOST A:
=======

**** IP: 172.17.8.101 ****

**** Output from routing table ****

root@core-01:/# ip route
default via 10.0.2.2 dev eth0  proto dhcp  src 10.0.2.15  metric 1024
10.0.2.0/24 dev eth0  proto kernel  scope link  src 10.0.2.15
10.0.2.2 dev eth0  proto dhcp  scope link  src 10.0.2.15  metric 1024
10.1.0.0/16 dev docker0  proto kernel  scope link  src 10.1.42.1
172.17.8.0/24 dev eth1  proto kernel  scope link  src 172.17.8.101
192.168.1.1 dev cali1fb91036fff  scope link
192.168.1.2 dev cali1ff66dc8fff  scope link
192.168.1.3 dev cali2078697cfff  scope link
192.168.1.4 via 172.17.8.102 dev eth1  proto bird
192.168.1.5 via 172.17.8.102 dev eth1  proto bird

**** Bird command line ****

 bird -R -s bird.ctl -d -c /config/bird.cfg

**** Bird config ****

router id 172.17.8.101;
log "/var/log/calico/bird.log" all;

filter calico_pools {
    if ( net ~ 192.168.0.0/16 ) then {
        accept;
    }
    else {
        reject;
    }
}

protocol kernel {
  learn;          # Learn all alien routes from the kernel
  persist;        # Don't remove routes on bird shutdown
  scan time 2;    # Scan kernel routing table every 2 seconds
  import all;
  device routes;
  export all;     # Default is export none
  graceful restart;
}

protocol device {
  scan time 2;    # Scan interfaces every 2 seconds
}

protocol direct {
   debug all;
   interface -"cali*", "*"; # Exclude cali* but include everything else.
}

# Template for all BGP clients
template bgp bgp_template {
  debug all;
  description "Connection to BGP peer";
  local as 64511;
  multihop;
  gateway recursive;
  import all;
  export filter calico_pools;
  next hop self;
  source address 172.17.8.101;
  add paths on;
  graceful restart;
}

protocol bgp from bgp_template {
  neighbor 172.17.8.102 as 64511;
}

HOST B (pretty much identical to HOST A with IPs swapped):
=======

**** IP: 172.17.8.102 ****

**** Output from routing table ****

root@core-02:/# ip route
default via 10.0.2.2 dev eth0  proto dhcp  src 10.0.2.15  metric 1024
10.0.2.0/24 dev eth0  proto kernel  scope link  src 10.0.2.15
10.0.2.2 dev eth0  proto dhcp  scope link  src 10.0.2.15  metric 1024
10.1.0.0/16 dev docker0  proto kernel  scope link  src 10.1.42.1
172.17.8.0/24 dev eth1  proto kernel  scope link  src 172.17.8.102
192.168.1.1 via 172.17.8.101 dev eth1  proto bird
192.168.1.2 via 172.17.8.101 dev eth1  proto bird
192.168.1.3 via 172.17.8.101 dev eth1  proto bird
192.168.1.4 dev cali32d26ef6fff  scope link
192.168.1.5 dev cali333fbba0fff  scope link

**** Bird command line ****

 bird -R -s bird.ctl -d -c /config/bird.cfg

**** Bird config ****

router id 172.17.8.102;
log "/var/log/calico/bird.log" all;

filter calico_pools {
    if ( net ~ 192.168.0.0/16 ) then {
        accept;
    }
    else {
    reject;
    }
}

protocol kernel {
  debug all;
  learn;          # Learn all alien routes from the kernel
  persist;        # Don't remove routes on bird shutdown
  scan time 2;    # Scan kernel routing table every 2 seconds
  import all;
  device routes;
  export all;     # Default is export none
  graceful restart;
}

protocol device {
  scan time 2;    # Scan interfaces every 2 seconds
}

protocol direct {
   debug all;
   interface -"cali*", "*"; # Exclude cali* but include everything else.
}

template bgp bgp_template {
  debug all;
  description "Connection to BGP peer";

  local as 64511;
  multihop;
  gateway recursive;
  import all;
  export filter calico_pools;
  next hop self;
  source address 172.17.8.102;  # The local address we use for the TCP
connection
  add paths on;
  graceful restart;
}

protocol bgp from bgp_template {
  neighbor 172.17.8.101 as 64511;
}

LOG FILE
========

If I send in a SIGHUP to bird on HOST B, I lose the routes to the
192.168.1.4 and .5 on HOST A, and the routes to 192.168.1.1, .2 and .3 on
HOST B.

Output from the logs on HOST B indicates that these routes are being
explicitly removed.  Given we have GR turned on, and are persisting kernel
routes, I would not have expected this.

Logs from HOST B as follow:

2015-05-21 22:52:54 <INFO> Reconfiguration requested by SIGHUP
2015-05-21 22:52:54 <INFO> Reconfiguring
2015-05-21 22:52:54 <TRACE> kernel1: Reconfigured
2015-05-21 22:52:54 <TRACE> direct1: Reconfigured
2015-05-21 22:52:54 <INFO> Removing protocol bgp1
2015-05-21 22:52:54 <TRACE> bgp1: Shutting down
2015-05-21 22:52:54 <TRACE> bgp1: Shutdown requested
2015-05-21 22:52:54 <TRACE> bgp1: State changed to stop
2015-05-21 22:52:54 <TRACE> bgp1: BGP session closed
2015-05-21 22:52:54 <INFO> Adding protocol bgp1
2015-05-21 22:52:54 <TRACE> bgp1: Initializing
2015-05-21 22:52:54 <TRACE> bgp1: Starting
2015-05-21 22:52:54 <TRACE> bgp1: State changed to start
2015-05-21 22:52:54 <TRACE> bgp1 > removed [sole] 192.168.1.1/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> kernel1 < removed 192.168.1.1/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> bgp1 > removed [sole] 192.168.1.2/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> kernel1 < removed 192.168.1.2/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> bgp1 > removed [sole] 192.168.1.3/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> kernel1 < removed 192.168.1.3/32 via
172.17.8.101 on eth1
2015-05-21 22:52:54 <TRACE> bgp1: Sending NOTIFICATION(code=6.3)
2015-05-21 22:52:54 <TRACE> bgp1: Down
2015-05-21 22:52:54 <TRACE> bgp1: State changed to down
2015-05-21 22:52:54 <TRACE> bgp1: Started
2015-05-21 22:52:54 <TRACE> bgp1: Connect delayed by 5 seconds
2015-05-21 22:52:54 <INFO> Reconfigured
2015-05-21 22:52:56 <TRACE> kernel1: Scanning routing table
2015-05-21 22:52:56 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:52:56 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:52:56 <TRACE> ...
2015-05-21 22:52:56 <TRACE> kernel1: Pruning table master
2015-05-21 22:52:56 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:52:56 <WARN> Netlink: File exists
2015-05-21 22:52:56 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:52:56 <WARN> Netlink: File exists
2015-05-21 22:52:56 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:52:56 <WARN> ...
2015-05-21 22:52:56 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:52:58 <TRACE> kernel1: Scanning routing table
2015-05-21 22:52:58 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:52:58 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:52:58 <TRACE> ...
2015-05-21 22:52:58 <TRACE> kernel1: Pruning table master
2015-05-21 22:52:58 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:52:58 <WARN> Netlink: File exists
2015-05-21 22:52:58 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:52:58 <WARN> Netlink: File exists
2015-05-21 22:52:58 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:52:58 <WARN> ...
2015-05-21 22:52:58 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:52:59 <TRACE> bgp1: Incoming connection from 172.17.8.101
(port 59189) accepted
2015-05-21 22:52:59 <TRACE> bgp1: Sending
OPEN(ver=4,as=64511,hold=240,id=ac110866)
2015-05-21 22:52:59 <TRACE> bgp1: Got OPEN(as=64511,hold=240,id=ac110865)
2015-05-21 22:52:59 <TRACE> bgp1: Sending KEEPALIVE
2015-05-21 22:52:59 <TRACE> bgp1: Got KEEPALIVE
2015-05-21 22:52:59 <TRACE> bgp1: BGP session established
2015-05-21 22:52:59 <TRACE> bgp1: Connected to table master
2015-05-21 22:52:59 <TRACE> bgp1: State changed to feed
2015-05-21 22:52:59 <TRACE> bgp1 < filtered out 0.0.0.0/0 via 10.0.2.2 on
eth0
2015-05-21 22:52:59 <TRACE> bgp1 < filtered out 10.0.2.2/32 dev eth0
2015-05-21 22:52:59 <TRACE> bgp1 < filtered out 10.0.2.0/24 dev eth0
2015-05-21 22:52:59 <TRACE> bgp1 < filtered out 10.1.0.0/16 dev docker0
2015-05-21 22:52:59 <TRACE> bgp1 < added 192.168.1.4/32 dev cali32d26ef6fff
2015-05-21 22:52:59 <TRACE> bgp1 < added 192.168.1.5/32 dev cali333fbba0fff
2015-05-21 22:52:59 <TRACE> bgp1 < filtered out 172.17.8.0/24 dev eth1
2015-05-21 22:52:59 <TRACE> bgp1: State changed to up
2015-05-21 22:52:59 <TRACE> bgp1: Sending UPDATE
2015-05-21 22:52:59 <TRACE> bgp1: Sending END-OF-RIB
2015-05-21 22:52:59 <TRACE> bgp1: Got UPDATE
2015-05-21 22:52:59 <TRACE> bgp1 > added [best] 192.168.1.1/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> bgp1 < rejected by protocol 192.168.1.1/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> kernel1 < added 192.168.1.1/32 via 172.17.8.101
on eth1
2015-05-21 22:52:59 <TRACE> bgp1 > added [best] 192.168.1.2/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> bgp1 < rejected by protocol 192.168.1.2/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> kernel1 < added 192.168.1.2/32 via 172.17.8.101
on eth1
2015-05-21 22:52:59 <TRACE> bgp1 > added [best] 192.168.1.3/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> bgp1 < rejected by protocol 192.168.1.3/32 via
172.17.8.101 on eth1
2015-05-21 22:52:59 <TRACE> kernel1 < added 192.168.1.3/32 via 172.17.8.101
on eth1
2015-05-21 22:52:59 <TRACE> bgp1: Got UPDATE
2015-05-21 22:52:59 <TRACE> bgp1: Got END-OF-RIB
2015-05-21 22:53:00 <TRACE> kernel1: Scanning routing table
2015-05-21 22:53:00 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:53:00 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:53:00 <TRACE> kernel1: 192.168.1.1/32: seen
2015-05-21 22:53:00 <TRACE> kernel1: 192.168.1.2/32: seen
2015-05-21 22:53:00 <TRACE> kernel1: 192.168.1.3/32: seen
2015-05-21 22:53:00 <TRACE> ...
2015-05-21 22:53:00 <TRACE> kernel1: Pruning table master
2015-05-21 22:53:00 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:53:00 <WARN> Netlink: File exists
2015-05-21 22:53:00 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:53:00 <WARN> Netlink: File exists
2015-05-21 22:53:00 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:53:00 <WARN> ...
2015-05-21 22:53:00 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:53:02 <TRACE> kernel1: Scanning routing table
2015-05-21 22:53:02 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:53:02 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:53:02 <TRACE> kernel1: 192.168.1.1/32: seen
2015-05-21 22:53:02 <TRACE> kernel1: 192.168.1.2/32: seen
2015-05-21 22:53:02 <TRACE> kernel1: 192.168.1.3/32: seen
2015-05-21 22:53:02 <TRACE> ...
2015-05-21 22:53:02 <TRACE> kernel1: Pruning table master
2015-05-21 22:53:02 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:53:02 <WARN> Netlink: File exists
2015-05-21 22:53:02 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:53:02 <WARN> Netlink: File exists
2015-05-21 22:53:02 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:53:02 <WARN> ...
2015-05-21 22:53:02 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:53:04 <TRACE> kernel1: Scanning routing table
2015-05-21 22:53:04 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:53:04 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:53:04 <TRACE> kernel1: 192.168.1.1/32: seen
2015-05-21 22:53:04 <TRACE> kernel1: 192.168.1.2/32: seen
2015-05-21 22:53:04 <TRACE> kernel1: 192.168.1.3/32: seen
2015-05-21 22:53:04 <TRACE> ...
2015-05-21 22:53:04 <TRACE> kernel1: Pruning table master
2015-05-21 22:53:04 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:53:04 <WARN> Netlink: File exists
2015-05-21 22:53:04 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:53:04 <WARN> Netlink: File exists
2015-05-21 22:53:04 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:53:04 <WARN> ...
2015-05-21 22:53:04 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:53:06 <TRACE> kernel1: Scanning routing table
2015-05-21 22:53:06 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:53:06 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:53:06 <TRACE> kernel1: 192.168.1.1/32: seen
2015-05-21 22:53:06 <TRACE> kernel1: 192.168.1.2/32: seen
2015-05-21 22:53:06 <TRACE> kernel1: 192.168.1.3/32: seen
2015-05-21 22:53:06 <TRACE> ...
2015-05-21 22:53:06 <TRACE> kernel1: Pruning table master
2015-05-21 22:53:06 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:53:06 <WARN> Netlink: File exists
2015-05-21 22:53:06 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:53:06 <WARN> Netlink: File exists
2015-05-21 22:53:06 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:53:06 <WARN> ...
2015-05-21 22:53:06 <TRACE> kernel1: Pruning inherited routes
2015-05-21 22:53:08 <TRACE> kernel1: Scanning routing table
2015-05-21 22:53:08 <TRACE> kernel1: 0.0.0.0/0: [alien] seen
2015-05-21 22:53:08 <TRACE> kernel1: 10.0.2.2/32: [alien] seen
2015-05-21 22:53:08 <TRACE> kernel1: 192.168.1.1/32: seen
2015-05-21 22:53:08 <TRACE> kernel1: 192.168.1.2/32: seen
2015-05-21 22:53:08 <TRACE> kernel1: 192.168.1.3/32: seen
2015-05-21 22:53:08 <TRACE> ...
2015-05-21 22:53:08 <TRACE> kernel1: Pruning table master
2015-05-21 22:53:08 <TRACE> kernel1: 10.0.2.0/24: reinstalling
2015-05-21 22:53:08 <WARN> Netlink: File exists
2015-05-21 22:53:08 <TRACE> kernel1: 10.1.0.0/16: reinstalling
2015-05-21 22:53:08 <WARN> Netlink: File exists
2015-05-21 22:53:08 <TRACE> kernel1: 172.17.8.0/24: reinstalling
2015-05-21 22:53:08 <WARN> ...
2015-05-21 22:53:08 <TRACE> kernel1: Pruning inherited routes

Routes disappear after reloading BIRD configuration

Robert Brockbank