HA support for DVR centralized default SNAT functionality on RDO Mitaka Milestone 3

Verification been done bellow is actually targeting conversion of HAProxy/Keepalived (Active/Active) 3 Node Controller which design was suggested for RDO Liberty  in https://github.com/beekhof/osp-ha-deploy/blob/master/HA-keepalived.md
to be able support Compute Nodes running in DVR mode. The core issue on Liberty was resolved for Mitaka , see upstream record  [RFE] Unable to create a router that's both HA and distributed 
General concepts (DVR/SNAT) are explained here Distributed Virtual Routing – SNAT

Original RDO Mitaka M3 four nodes deployment :-

ServerCentOS01  - Controller Node 192.169.142.127 (MGMT NET)
ServerCentOS02  - Network Node    192.169.142.147 (MGMT NET)
ServerCentOS03  - Network Node    192.169.142.157 (MGMT NET)
ServerCentOS04  - Compute Node    192.169.142.137 (MGMT NET)

Per https://www.rdoproject.org/testday/mitaka/milestone3/

Install the yum-plugin-priorities package

• # yum -y install yum-plugin-priorities
  
• For CentOS 7 and RHEL 7, install the required .repo files:
  # cd /etc/yum.repos.d/
  # curl -O http://trunk.rdoproject.org/centos7/delorean-deps.repo
  # curl -O http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo
• On Controller `yum -y install openstack-packstack` 

****************
Answer file 
****************
[root@ServerCenttOS01 ~]# cat  answerDVR_SNAT.txt
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_SERVICE_WORKERS=%{::processorcount}
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_MANILA_INSTALL=n
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=n
CONFIG_CEILOMETER_INSTALL=y
CONFIG_AODH_INSTALL=y
CONFIG_GNOCCHI_INSTALL=y
CONFIG_SAHARA_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_TROVE_INSTALL=n
CONFIG_IRONIC_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137
CONFIG_NETWORK_HOSTS=192.169.142.147,192.169.142.157
CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_USE_SUBNETS=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAMES=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_SAHARA_HOST=192.169.142.127
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_ENABLE_RDO_TESTING=n
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_SAT6_SERVER=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_SAT6_ORG=
CONFIG_RH_SAT6_KEY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_SSL_CACERT_FILE=/etc/pki/tls/certs/selfcert.crt
CONFIG_SSL_CACERT_KEY_FILE=/etc/pki/tls/private/selfkey.key
CONFIG_SSL_CERT_DIR=~/packstackca/
CONFIG_SSL_CACERT_SELFSIGN=y
CONFIG_SELFSIGN_CACERT_SUBJECT_C=--
CONFIG_SELFSIGN_CACERT_SUBJECT_ST=State
CONFIG_SELFSIGN_CACERT_SUBJECT_L=City
CONFIG_SELFSIGN_CACERT_SUBJECT_O=openstack
CONFIG_SELFSIGN_CACERT_SUBJECT_OU=packstack
CONFIG_SELFSIGN_CACERT_SUBJECT_CN=ip-192-169-142-127.ip.secureserver.net
CONFIG_SELFSIGN_CACERT_SUBJECT_MAIL=admin@ip-192-169-142-127.ip.secureserver.net
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=7207ae344ed04957
CONFIG_KEYSTONE_DB_PW=abcae16b785245c3
CONFIG_KEYSTONE_DB_PURGE_ENABLE=True
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=3ad2de159f9649afb0c342ba57e637d9
CONFIG_KEYSTONE_ADMIN_EMAIL=root@localhost
CONFIG_KEYSTONE_ADMIN_USERNAME=admin
CONFIG_KEYSTONE_ADMIN_PW=7049f834927e4468
CONFIG_KEYSTONE_DEMO_PW=bf737b785cfa4398
CONFIG_KEYSTONE_API_VERSION=v2.0
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_KEYSTONE_IDENTITY_BACKEND=sql
CONFIG_KEYSTONE_LDAP_URL=ldap://192.169.142.127
CONFIG_KEYSTONE_LDAP_USER_DN=
CONFIG_KEYSTONE_LDAP_USER_PASSWORD=
CONFIG_KEYSTONE_LDAP_SUFFIX=
CONFIG_KEYSTONE_LDAP_QUERY_SCOPE=one
CONFIG_KEYSTONE_LDAP_PAGE_SIZE=-1
CONFIG_KEYSTONE_LDAP_USER_SUBTREE=
CONFIG_KEYSTONE_LDAP_USER_FILTER=
CONFIG_KEYSTONE_LDAP_USER_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_USER_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_MAIL_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_MASK=-1
CONFIG_KEYSTONE_LDAP_USER_ENABLED_DEFAULT=TRUE
CONFIG_KEYSTONE_LDAP_USER_ENABLED_INVERT=n
CONFIG_KEYSTONE_LDAP_USER_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_USER_DEFAULT_PROJECT_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_USER_PASS_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_EMULATION_DN=
CONFIG_KEYSTONE_LDAP_USER_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_GROUP_SUBTREE=
CONFIG_KEYSTONE_LDAP_GROUP_FILTER=
CONFIG_KEYSTONE_LDAP_GROUP_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_GROUP_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_MEMBER_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_DESC_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_GROUP_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_USE_TLS=n
CONFIG_KEYSTONE_LDAP_TLS_CACERTDIR=
CONFIG_KEYSTONE_LDAP_TLS_CACERTFILE=
CONFIG_KEYSTONE_LDAP_TLS_REQ_CERT=demand
CONFIG_GLANCE_DB_PW=41264fc52ffd4fe8
CONFIG_GLANCE_KS_PW=f6a9398960534797
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=5ac08c6d09ba4b69
CONFIG_CINDER_DB_PURGE_ENABLE=True
CONFIG_CINDER_KS_PW=c8cb1ecb8c2b4f6f
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=5G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES=
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=/etc/cinder/shares.conf
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_PARTNER_BACKEND_NAME=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_ESERIES_HOST_TYPE=linux_dm_mp
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_IRONIC_DB_PW=PW_PLACEHOLDER
CONFIG_IRONIC_KS_PW=PW_PLACEHOLDER
CONFIG_NOVA_DB_PURGE_ENABLE=True
CONFIG_NOVA_DB_PW=1e1b5aeeeaf342a8
CONFIG_NOVA_KS_PW=d9583177a2444f06
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_MANAGER=nova.compute.manager.ComputeManager
CONFIG_VNC_SSL_CERT=
CONFIG_VNC_SSL_KEY=
CONFIG_NOVA_PCI_ALIAS=
CONFIG_NOVA_PCI_PASSTHROUGH_WHITELIST=
CONFIG_NOVA_COMPUTE_PRIVIF=
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=808e36e154bd4cee
CONFIG_NEUTRON_DB_PW=0e2b927a21b44737
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_METADATA_PW=a965cd23ed2f4502
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_VPNAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_ML2_SUPPORTED_PCI_VENDOR_DEVS=['15b3:1004', '8086:10ca']
CONFIG_NEUTRON_ML2_SRIOV_AGENT_REQUIRED=n
CONFIG_NEUTRON_ML2_SRIOV_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
CONFIG_NEUTRON_OVS_TUNNEL_SUBNETS=
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_MANILA_DB_PW=PW_PLACEHOLDER
CONFIG_MANILA_KS_PW=PW_PLACEHOLDER
CONFIG_MANILA_BACKEND=generic
CONFIG_MANILA_NETAPP_DRV_HANDLES_SHARE_SERVERS=false
CONFIG_MANILA_NETAPP_TRANSPORT_TYPE=https
CONFIG_MANILA_NETAPP_LOGIN=admin
CONFIG_MANILA_NETAPP_PASSWORD=
CONFIG_MANILA_NETAPP_SERVER_HOSTNAME=
CONFIG_MANILA_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_MANILA_NETAPP_SERVER_PORT=443
CONFIG_MANILA_NETAPP_AGGREGATE_NAME_SEARCH_PATTERN=(.*)
CONFIG_MANILA_NETAPP_ROOT_VOLUME_AGGREGATE=
CONFIG_MANILA_NETAPP_ROOT_VOLUME_NAME=root
CONFIG_MANILA_NETAPP_VSERVER=
CONFIG_MANILA_GENERIC_DRV_HANDLES_SHARE_SERVERS=true
CONFIG_MANILA_GENERIC_VOLUME_NAME_TEMPLATE=manila-share-%s
CONFIG_MANILA_GENERIC_SHARE_MOUNT_PATH=/shares
CONFIG_MANILA_SERVICE_IMAGE_LOCATION=https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2
CONFIG_MANILA_SERVICE_INSTANCE_USER=ubuntu
CONFIG_MANILA_SERVICE_INSTANCE_PASSWORD=ubuntu
CONFIG_MANILA_NETWORK_TYPE=neutron
CONFIG_MANILA_NETWORK_STANDALONE_GATEWAY=
CONFIG_MANILA_NETWORK_STANDALONE_NETMASK=
CONFIG_MANILA_NETWORK_STANDALONE_SEG_ID=
CONFIG_MANILA_NETWORK_STANDALONE_IP_RANGE=
CONFIG_MANILA_NETWORK_STANDALONE_IP_VERSION=4
CONFIG_MANILA_GLUSTERFS_SERVERS=
CONFIG_MANILA_GLUSTERFS_NATIVE_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_VOLUME_PATTERN=
CONFIG_MANILA_GLUSTERFS_TARGET=
CONFIG_MANILA_GLUSTERFS_MOUNT_POINT_BASE=
CONFIG_MANILA_GLUSTERFS_NFS_SERVER_TYPE=gluster
CONFIG_MANILA_GLUSTERFS_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_GANESHA_SERVER_IP=
CONFIG_HORIZON_SSL=n
CONFIG_HORIZON_SECRET_KEY=33cade531a764c858e4e6c22488f379f
CONFIG_HORIZON_SSL_CERT=
CONFIG_HORIZON_SSL_KEY=
CONFIG_HORIZON_SSL_CACERT=
CONFIG_SWIFT_KS_PW=PW_PLACEHOLDER
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=a940c9a54fbb4af8
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=36ce78ff06ef4577
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin
CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_IMAGE_NAME=cirros
CONFIG_PROVISION_IMAGE_URL=http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
CONFIG_PROVISION_IMAGE_FORMAT=qcow2
CONFIG_PROVISION_IMAGE_SSH_USER=cirros
CONFIG_TEMPEST_HOST=
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=PW_PLACEHOLDER
CONFIG_PROVISION_TEMPEST_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_RUN_TEMPEST=n
CONFIG_RUN_TEMPEST_TESTS=smoke
CONFIG_PROVISION_OVS_BRIDGE=n
CONFIG_CEILOMETER_SECRET=19ae0e7430174349
CONFIG_CEILOMETER_KS_PW=337b08d4b3a44753
CONFIG_CEILOMETER_COORDINATION_BACKEND=redis
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_REDIS_MASTER_HOST=192.169.142.127
CONFIG_REDIS_PORT=6379
CONFIG_REDIS_HA=n
CONFIG_REDIS_SLAVE_HOSTS=
CONFIG_REDIS_SENTINEL_HOSTS=
CONFIG_REDIS_SENTINEL_CONTACT_HOST=
CONFIG_REDIS_SENTINEL_PORT=26379
CONFIG_REDIS_SENTINEL_QUORUM=2
CONFIG_REDIS_MASTER_NAME=mymaster
CONFIG_AODH_KS_PW=acdd500a5fed4700
CONFIG_GNOCCHI_DB_PW=cf11b5d6205f40e7
CONFIG_GNOCCHI_KS_PW=36eba4690b224044
CONFIG_TROVE_DB_PW=PW_PLACEHOLDER
CONFIG_TROVE_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_NOVA_USER=trove
CONFIG_TROVE_NOVA_TENANT=services
CONFIG_TROVE_NOVA_PW=PW_PLACEHOLDER
CONFIG_SAHARA_DB_PW=PW_PLACEHOLDER
CONFIG_SAHARA_KS_PW=PW_PLACEHOLDER
CONFIG_NAGIOS_PW=02f168ee8edd44e4

Up on completion :-

[root@ServerCenttOS01 ~]# nova-manage version
13.0.0-0.20160304162843.c5a45a2.el7.centos

OVS external bridges activated on both  Network and Compute Node

***********************************************************
Upon completion on Network node 192.169.142.147
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.229"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

***********************************************************
On Network node 192.169.142.157
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.230"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

***********************************************************
On Compute node 192.169.142.137
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.231"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no


********************************************************************************
Next step was performed on both Network Nodes and Compute Node
********************************************************************************
#!/bin/bash -x
chkconfig network on
systemctl stop NetworkManager
systemctl disable NetworkManager
service network restart

At this point we start DVR/SNAT tuning

******************************************************************************************
On Controller an both Network Nodes  /etc/neutron/neutron.conf updated as follows
******************************************************************************************
dvr_base_mac = fa:16:3f:00:00:00

# System-wide flag to determine the type of router that tenants can create.
# Only admin can override. (boolean value)
router_distributed = True

# Enable HA mode for virtual routers. (boolean value)
l3_ha = True

******************************************************************************************
On  both Network Nodes  /etc/neutron/l3_agent.ini updated as follows
******************************************************************************************
[root@ServerCentOS02 neutron]# cat l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
agent_mode = dvr_snat
external_network_bridge = br-ex
debug = False
[AGENT]


Files /etc/neutron/plugins/ml2/ml2_conf.ini,openvswitch_agent.ini,l3_agent.ini tuned on Network and Compute Nodes exactly as it was done in
RDO Liberty DVR Neutron workflow on CentOS 7.2
/etc/neutron/metadata_agent.ini copied over to Compute from Network Node.

**********************************************************************************
On Compute node neutron-l3-agent and neutron-metadata-agent are
supposed to be started via script
**********************************************************************************
 #!/bin/bash -x
 yum install  openstack-neutron-ml2  -y ;
 systemctl start neutron-l3-agent ;
 systemctl start neutron-metadata-agent ;
 systemctl restart neutron-openvswitch-agent ;
 systemctl enable neutron-l3-agent ;
 systemctl enable neutron-metadata-agent

All nodes have been rebooted.


Router was created via dashboard as RouterDVS :-

[root@ServerCenttOS01 ~(keystone_admin)]# neutron router-show RouterDVS
+-------------------------+------------------------------------------------------------------------+
| Field                   | Value                                                                  |
+-------------------------+------------------------------------------------------------------------+
| admin_state_up          | True                                                                   |
| availability_zone_hints |                                                                        |
| availability_zones      | nova                                                                   |
| distributed             | True                                                                   |
| external_gateway_info   | {"network_id": "1c347a42-21fa-4273-ad17-fa210d546ffd", "enable_snat":  |
|                         | true, "external_fixed_ips": [{"subnet_id": "fd24fa1d-cd2a-             |
|                         | 4a80-a822-e0a2fa5f743a", "ip_address": "172.24.4.227"}]}               |
| ha                      | True                                                                   |
| id                      | dd0d0741-c8a1-465a-8f89-ad986cd0592f                                   |
| name                    | RouterDVS                                                              |
| routes                  |                                                                        |
| status                  | ACTIVE                                                                 |
| tenant_id               | 06f56a00961e4c3ea10b537df8c86e1b                                       |
+-------------------------+------------------------------------------------------------------------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+------------------------------+----------------------------+----------------+-------+----------+
| id                           | host                       | admin_state_up | alive | ha_state |
+------------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-     | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                 |                            |                |       |          |
| c96930fa-066c-               | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf       |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-     | ServerCentOS03.localdomain | True           | :-)   | standby  |
| ef4ba2fe4105                 |                            |                |       |          |
+------------------------------+----------------------------+----------------+-------+----------+

Per https://review.openstack.org/#/c/196893/
Instead of running in the qrouter namespace, keepalived will run inside the snat-namespace. Therefore only snat ports will fall under the control of the HA domain.

  

[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ps -ef | grep keepalived
neutron   3168     1  0 00:29 ?        00:00:00 /usr/bin/python2 /bin/neutron-keepalived-state-change --router_id=dd0d0741-c8a1-465a-8f89-ad986cd0592f --namespace=snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f --conf_dir=/var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f --monitor_interface=ha-f3a6b78f-5f --monitor_cidr=169.254.0.1/24 --pid_file=/var/lib/neutron/external/pids/dd0d0741-c8a1-465a-8f89-ad986cd0592f.monitor.pid --state_path=/var/lib/neutron --user=988 --group=983
root      3385     1  0 00:29 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      3386  3385  0 00:29 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      7853  7677  0 00:56 pts/1    00:00:00 grep --color=auto keepalived

[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ip route
default via 172.24.4.225 dev qg-a31b1c28-8d
50.0.0.0/24 dev sg-3015f2cd-a4  proto kernel  scope link  src 50.0.0.11
169.254.0.0/24 dev ha-f3a6b78f-5f  proto kernel  scope link  src 169.254.0.1
169.254.192.0/18 dev ha-f3a6b78f-5f  proto kernel  scope link  src 169.254.192.1
172.24.4.224/28 dev qg-a31b1c28-8d  proto kernel  scope link  src 172.24.4.227

[root@ServerCentOS02 ~]# ssh 192.169.142.157
root@192.169.142.157's password:
Last login: Sun Mar 13 00:30:59 2016

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ps -ef | grep keepalived
neutron   2997     1  0 00:30 ?        00:00:00 /usr/bin/python2 /bin/neutron-keepalived-state-change --router_id=dd0d0741-c8a1-465a-8f89-ad986cd0592f --namespace=snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f --conf_dir=/var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f --monitor_interface=ha-f9dd88a2-33 --monitor_cidr=169.254.0.1/24 --pid_file=/var/lib/neutron/external/pids/dd0d0741-c8a1-465a-8f89-ad986cd0592f.monitor.pid --state_path=/var/lib/neutron --user=988 --group=983
root      3216     1  0 00:30 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      3217  3216  0 00:30 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      7682  7614  0 00:58 pts/1    00:00:00 grep --color=auto keepalived

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f  ip route
169.254.192.0/18 dev ha-f9dd88a2-33  proto kernel  scope link  src 169.254.192.2

[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ifconfig
ha-f3a6b78f-5f: flags=4163  mtu 1450
        inet 169.254.192.1  netmask 255.255.192.0  broadcast 169.254.255.255
        inet6 fe80::f816:3eff:fec0:50ff  prefixlen 64  scopeid 0x20
        ether fa:16:3e:c0:50:ff  txqueuelen 0  (Ethernet)
        RX packets 8  bytes 684 (684.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1224  bytes 66336 (64.7 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-a31b1c28-8d: flags=4163  mtu 1450
        inet 172.24.4.227  netmask 255.255.255.240  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe4d:d973  prefixlen 64  scopeid 0x20
        ether fa:16:3e:4d:d9:73  txqueuelen 0  (Ethernet)
        RX packets 51  bytes 3981 (3.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 25  bytes 1910 (1.8 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

sg-3015f2cd-a4: flags=4163  mtu 1450
        inet 50.0.0.11  netmask 255.255.255.0  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe8c:dbd3  prefixlen 64  scopeid 0x20
        ether fa:16:3e:8c:db:d3  txqueuelen 0  (Ethernet)
        RX packets 15  bytes 1282 (1.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 26  bytes 2020 (1.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@ServerCentOS02 ~]# ssh 192.169.142.157
root@192.169.142.157's password:
Last login: Sun Mar 13 01:07:02 2016 from ip-192-169-142-147.ip.secureserver.net

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f  ifconfig
ha-f9dd88a2-33: flags=4163  mtu 1450
        inet 169.254.192.2  netmask 255.255.192.0  broadcast 169.254.255.255
        inet6 fe80::f816:3eff:fead:71  prefixlen 64  scopeid 0x20
        ether fa:16:3e:ad:00:71  txqueuelen 0  (Ethernet)
        RX packets 1215  bytes 65930 (64.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 11  bytes 954 (954.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-a31b1c28-8d: flags=4163  mtu 1450
        ether fa:16:3e:4d:d9:73  txqueuelen 0  (Ethernet)
        RX packets 54  bytes 4270 (4.1 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

sg-3015f2cd-a4: flags=4163  mtu 1450
        ether fa:16:3e:8c:db:d3  txqueuelen 0  (Ethernet)
        RX packets 63  bytes 3922 (3.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

Verification is done. 


[root@ServerCenttOS01 ~(keystone_admin)]# neutron net-list
+-------------------------------+-------------------------------+-------------------------------+
| id                            | name                          | subnets                       |
+-------------------------------+-------------------------------+-------------------------------+
| 1c347a42-21fa-4273-ad17-fa210 | public                        | fd24fa1d-cd2a-                |
| d546ffd                         |                               | 4a80-a822-e0a2fa5f743a        |
|                                     |              | 172.24.4.224/28   |  <== External Network
| 498a3600-0b40-49c0-8ec1-c4b95 | private                       | 33478000-2584-4b24-8f39-1482c |
| 5a4335e                       |                               | 5b853af 10.0.0.0/24           |
| 70034a53-52c8-4665-9ed1-2dc7d | HA network tenant 06f56a00961 | c2bbd68c-0d9d-                |
| 3380a98                       | e4c3ea10b537df8c86e1b         | 49b1-a270-e98bdd08783e        |
|                               |                               | 169.254.192.0/18              |
| 08607e5c-fc14-488d-9c9c-      | demo_network                  | ebd72d77-6ea2-4d4e-           |
| 4d5e14040a6e                  |                               | a5e2-650e745d3db6 50.0.0.0/24 |
+-------------------------------+-------------------------------+-------------------------------

******************************************************************************
During run-time Network Nodes have been randomly shutdown
*******************************************************************************
[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   | <=== Brought down
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | standby   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | xxx   | standby  |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | active   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | standby  |  <== Brought up again
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | active   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+



[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | xxx   | standby  | <== Brought  down
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | standby  | <=== Brought up again
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+