Monday, March 21, 2016

Attempt to set up RDO Mitaka (RC1) at any given time (Delorean trunks)

UPDATE 04/13/2016

Switching to newly added Storage node on RDO Mitaka, after adding new
Storage Node to cluster via packstack

*****************************************************************************
Keystone database still contains IP of old Storage Server (Glance)
*****************************************************************************
[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user  root --password=7207ae344ed04957  keystone -e "select interface, url from endpoint where service_id =  (select id from service where service.type = 'image');"

+-----------+-----------------------------+
| interface | url                         |
+-----------+-----------------------------+
| admin     | http://192.169.142.127:9292 |
| internal  | http://192.169.142.127:9292 |
| public    | http://192.169.142.127:9292 |
+-----------+-----------------------------+

**************************************************************************
Updating "url" field of endpoint table with IP of new Storage Server
***************************************************************************

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:9292' where  interface ='internal' and  service_id =  (select id from service where service.type = 'image');"

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:9292' where  interface ='public' and  service_id =  (select id from service where service.type = 'image');"

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:9292' where  interface ='admin' and  service_id =  (select id from service where service.type = 'image');"

***************************************
Make sure updates are done
***************************************

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user  root --password=7207ae344ed04957  keystone -e "select interface, url from endpoint where service_id =  (select id from service where service.type = 'image');"

+-----------+-----------------------------+
| interface | url                         |
+-----------+-----------------------------+
| admin     | http://192.169.142.117:9292 |
| internal  | http://192.169.142.117:9292 |
| public    | http://192.169.142.117:9292 |
+-----------+-----------------------------+

*****************************************************************************
Keystone database still contains IP of old Storage Server (Swift)
*****************************************************************************


[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user  root --password=7207ae344ed04957  keystone -e "select interface, url from endpoint where service_id =  (select id from service where service.type = 'object-store');"

+-----------+---------------------------------------------------+
| interface | url                                               |
+-----------+---------------------------------------------------+
| internal  | http://192.169.142.127:8080/v1/AUTH_%(tenant_id)s |
| public    | http://192.169.142.127:8080/v1/AUTH_%(tenant_id)s |
| admin     | http://192.169.142.127:8080/v1/AUTH_%(tenant_id)s |
+-----------+---------------------------------------------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s' where  interface ='admin' and  service_id =  (select id from service where service.type = 'object-store');"

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s' where  interface ='internal' and  service_id =  (select id from service where service.type = 'object-store');"

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user root --password=7207ae344ed04957  keystone -e "update endpoint set   url  = 'http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s' where  interface ='public' and  service_id =  (select id from service where service.type = 'object-store');"

******************************************
 Make sure updates are done
******************************************

[root@ip-192-169-142-127 ~(keystone_admin)]# mysql  --user  root --password=7207ae344ed04957  keystone -e "select interface, url from endpoint where service_id =  (select id from service where service.type = 'object-store');"

+-----------+---------------------------------------------------+
| interface | url                                             |
+-----------+---------------------------------------------------+
| internal | http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s |
| public    | http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s |
| admin    | http://192.169.142.117:8080/v1/AUTH_%(tenant_id)s |
+-----------+---------------------------------------------------+


# service httpd restart

END UPDATE


Quoting  Official delorean documentation

"The RDO project has a continuous integration pipeline that consists of multiple jobs that deploy and test OpenStack as accomplished by different installers. This vast test coverage attempts to ensure that there are no known issues either in packaging, in code or in the installers themselves.Once a Delorean consistent repository has undergone these tests successfully, it will be promoted to current-passed-ci. Current-passed-ci represents the latest and greatest
version of RDO trunk packages that were tested together successfully"


Set up current-passed-ci repositories on all deployment nodes Controller,Storage,Compute. It might be not really needed ( if packstack at run-time copies repositories from Controller to other nodes), but won't hurt anyway.

# yum -y install yum-plugin-priorities
# cd /etc/yum.repos.d
# curl -O https://trunk.rdoproject.org/centos7-mitaka/delorean-deps.repo
# curl -O https://trunk.rdoproject.org/centos7-mitaka/current-passed-ci/delorean.repo

******************
On Controller
******************

# yum -y install openstack-packstack

[root@SeverMitaka01 ~]# rpm -qa \*openstack-packstack\*
openstack-packstack-puppet-8.0.0-0.20160316101826.9fd26e8.el7.centos.noarch
openstack-packstack-8.0.0-0.20160316101826.9fd26e8.el7.centos.noarch

*******************************************************************************
Answer file for testing  mentioned 3 node deployment is here
*******************************************************************************

Two deployments done bellow intentionally test ability to add Storage Node
and hack keystone endpoint table to switch to new Swift Server


First test is simplest  two node cluster (Controller,Compute) install completed



*****************************************************************************
   Final configuration was obtained after adding Storage Node using
   EXCLUDE_SERVERS=192.169.142.127,192.169.142.137
   . . . . . . .
   CONFIG_UNSUPPORTED=y
   . . . . . . .
   CONFIG_STORAGE_HOST=192.169.142.157
   . . . . . . .  
   CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
   CONFIG_NEUTRON_OVS_TUNNEL_SUBNETS=12.0.0.0/24
 
  and updating keystone endpoint right inside keystone database
   for swift-proxy pointing to IP of added Server instead of  Controller
   been used for simplest two node Cluster test.
   Running packstack with answer-file posted at link "here" is supposed
   to create three node deployment via single run and create correct
   endpoints for all storage services  glance,cinder,swift  pointing  to
   192.169.142.157.
     So, CONFIG_UNSUPPORTED=y seems to work for oncoming RDO Mitaka
   release, but requires a bit more efforts as originally expected.
*****************************************************************************


    Second test  running to add Storage Node using    
    EXCLUDE_SERVERS=Controller-IP,Compute-IP


 

   Now hack keystone database to get new IP set for swift-proxy  record in endpoint table of keystone database , followed by `openstack-service restart` on Controller

  
  
   All set to use newly added Swift Node with 3 XFS 10GB drives involved
   in swift's replication as back end for Glance.


Thursday, March 10, 2016

HA support for DVR centralized default SNAT functionality on RDO Mitaka Milestone 3

Verification been done bellow is actually targeting conversion of HAProxy/Keepalived (Active/Active) 3 Node Controller which design was suggested for RDO Liberty  in https://github.com/beekhof/osp-ha-deploy/blob/master/HA-keepalived.md
to be able support Compute Nodes running in DVR mode. The core issue on Liberty was resolved for Mitaka , see upstream record  [RFE] Unable to create a router that's both HA and distributed 
General concepts (DVR/SNAT) are explained here Distributed Virtual Routing – SNAT

Original RDO Mitaka M3 four nodes deployment :-

ServerCentOS01  - Controller Node 192.169.142.127 (MGMT NET)
ServerCentOS02  - Network Node    192.169.142.147 (MGMT NET)
ServerCentOS03  - Network Node    192.169.142.157 (MGMT NET)
ServerCentOS04  - Compute Node    192.169.142.137 (MGMT NET)

Per https://www.rdoproject.org/testday/mitaka/milestone3/

Install the yum-plugin-priorities package
  • # yum -y install yum-plugin-priorities
  • For CentOS 7 and RHEL 7, install the required .repo files:
    # cd /etc/yum.repos.d/
    # curl -O http://trunk.rdoproject.org/centos7/delorean-deps.repo
    # curl -O http://trunk.rdoproject.org/centos7/current-passed-ci/delorean.repo
  • On Controller `yum -y install openstack-packstack` 

****************
Answer file 
****************
[root@ServerCenttOS01 ~]# cat  answerDVR_SNAT.txt
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_SERVICE_WORKERS=%{::processorcount}
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_MANILA_INSTALL=n
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=n
CONFIG_CEILOMETER_INSTALL=y
CONFIG_AODH_INSTALL=y
CONFIG_GNOCCHI_INSTALL=y
CONFIG_SAHARA_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_TROVE_INSTALL=n
CONFIG_IRONIC_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137
CONFIG_NETWORK_HOSTS=192.169.142.147,192.169.142.157

CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_USE_SUBNETS=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAMES=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_SAHARA_HOST=192.169.142.127
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_ENABLE_RDO_TESTING=n
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_SAT6_SERVER=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_SAT6_ORG=
CONFIG_RH_SAT6_KEY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_SSL_CACERT_FILE=/etc/pki/tls/certs/selfcert.crt
CONFIG_SSL_CACERT_KEY_FILE=/etc/pki/tls/private/selfkey.key
CONFIG_SSL_CERT_DIR=~/packstackca/
CONFIG_SSL_CACERT_SELFSIGN=y
CONFIG_SELFSIGN_CACERT_SUBJECT_C=--
CONFIG_SELFSIGN_CACERT_SUBJECT_ST=State
CONFIG_SELFSIGN_CACERT_SUBJECT_L=City
CONFIG_SELFSIGN_CACERT_SUBJECT_O=openstack
CONFIG_SELFSIGN_CACERT_SUBJECT_OU=packstack
CONFIG_SELFSIGN_CACERT_SUBJECT_CN=ip-192-169-142-127.ip.secureserver.net
CONFIG_SELFSIGN_CACERT_SUBJECT_MAIL=admin@ip-192-169-142-127.ip.secureserver.net
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=7207ae344ed04957
CONFIG_KEYSTONE_DB_PW=abcae16b785245c3
CONFIG_KEYSTONE_DB_PURGE_ENABLE=True
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=3ad2de159f9649afb0c342ba57e637d9
CONFIG_KEYSTONE_ADMIN_EMAIL=root@localhost
CONFIG_KEYSTONE_ADMIN_USERNAME=admin
CONFIG_KEYSTONE_ADMIN_PW=7049f834927e4468
CONFIG_KEYSTONE_DEMO_PW=bf737b785cfa4398
CONFIG_KEYSTONE_API_VERSION=v2.0
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_KEYSTONE_IDENTITY_BACKEND=sql
CONFIG_KEYSTONE_LDAP_URL=ldap://192.169.142.127
CONFIG_KEYSTONE_LDAP_USER_DN=
CONFIG_KEYSTONE_LDAP_USER_PASSWORD=
CONFIG_KEYSTONE_LDAP_SUFFIX=
CONFIG_KEYSTONE_LDAP_QUERY_SCOPE=one
CONFIG_KEYSTONE_LDAP_PAGE_SIZE=-1
CONFIG_KEYSTONE_LDAP_USER_SUBTREE=
CONFIG_KEYSTONE_LDAP_USER_FILTER=
CONFIG_KEYSTONE_LDAP_USER_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_USER_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_MAIL_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_MASK=-1
CONFIG_KEYSTONE_LDAP_USER_ENABLED_DEFAULT=TRUE
CONFIG_KEYSTONE_LDAP_USER_ENABLED_INVERT=n
CONFIG_KEYSTONE_LDAP_USER_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_USER_DEFAULT_PROJECT_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_USER_PASS_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_EMULATION_DN=
CONFIG_KEYSTONE_LDAP_USER_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_GROUP_SUBTREE=
CONFIG_KEYSTONE_LDAP_GROUP_FILTER=
CONFIG_KEYSTONE_LDAP_GROUP_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_GROUP_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_MEMBER_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_DESC_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_GROUP_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_USE_TLS=n
CONFIG_KEYSTONE_LDAP_TLS_CACERTDIR=
CONFIG_KEYSTONE_LDAP_TLS_CACERTFILE=
CONFIG_KEYSTONE_LDAP_TLS_REQ_CERT=demand
CONFIG_GLANCE_DB_PW=41264fc52ffd4fe8
CONFIG_GLANCE_KS_PW=f6a9398960534797
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=5ac08c6d09ba4b69
CONFIG_CINDER_DB_PURGE_ENABLE=True
CONFIG_CINDER_KS_PW=c8cb1ecb8c2b4f6f
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=5G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES=
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=/etc/cinder/shares.conf
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_PARTNER_BACKEND_NAME=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_ESERIES_HOST_TYPE=linux_dm_mp
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_IRONIC_DB_PW=PW_PLACEHOLDER
CONFIG_IRONIC_KS_PW=PW_PLACEHOLDER
CONFIG_NOVA_DB_PURGE_ENABLE=True
CONFIG_NOVA_DB_PW=1e1b5aeeeaf342a8
CONFIG_NOVA_KS_PW=d9583177a2444f06
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_MANAGER=nova.compute.manager.ComputeManager
CONFIG_VNC_SSL_CERT=
CONFIG_VNC_SSL_KEY=
CONFIG_NOVA_PCI_ALIAS=
CONFIG_NOVA_PCI_PASSTHROUGH_WHITELIST=
CONFIG_NOVA_COMPUTE_PRIVIF=
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=808e36e154bd4cee
CONFIG_NEUTRON_DB_PW=0e2b927a21b44737
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_METADATA_PW=a965cd23ed2f4502
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_VPNAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch

CONFIG_NEUTRON_ML2_SUPPORTED_PCI_VENDOR_DEVS=['15b3:1004', '8086:10ca']
CONFIG_NEUTRON_ML2_SRIOV_AGENT_REQUIRED=n
CONFIG_NEUTRON_ML2_SRIOV_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
CONFIG_NEUTRON_OVS_TUNNEL_SUBNETS=
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_MANILA_DB_PW=PW_PLACEHOLDER
CONFIG_MANILA_KS_PW=PW_PLACEHOLDER
CONFIG_MANILA_BACKEND=generic
CONFIG_MANILA_NETAPP_DRV_HANDLES_SHARE_SERVERS=false
CONFIG_MANILA_NETAPP_TRANSPORT_TYPE=https
CONFIG_MANILA_NETAPP_LOGIN=admin
CONFIG_MANILA_NETAPP_PASSWORD=
CONFIG_MANILA_NETAPP_SERVER_HOSTNAME=
CONFIG_MANILA_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_MANILA_NETAPP_SERVER_PORT=443
CONFIG_MANILA_NETAPP_AGGREGATE_NAME_SEARCH_PATTERN=(.*)
CONFIG_MANILA_NETAPP_ROOT_VOLUME_AGGREGATE=
CONFIG_MANILA_NETAPP_ROOT_VOLUME_NAME=root
CONFIG_MANILA_NETAPP_VSERVER=
CONFIG_MANILA_GENERIC_DRV_HANDLES_SHARE_SERVERS=true
CONFIG_MANILA_GENERIC_VOLUME_NAME_TEMPLATE=manila-share-%s
CONFIG_MANILA_GENERIC_SHARE_MOUNT_PATH=/shares
CONFIG_MANILA_SERVICE_IMAGE_LOCATION=https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2
CONFIG_MANILA_SERVICE_INSTANCE_USER=ubuntu
CONFIG_MANILA_SERVICE_INSTANCE_PASSWORD=ubuntu
CONFIG_MANILA_NETWORK_TYPE=neutron
CONFIG_MANILA_NETWORK_STANDALONE_GATEWAY=
CONFIG_MANILA_NETWORK_STANDALONE_NETMASK=
CONFIG_MANILA_NETWORK_STANDALONE_SEG_ID=
CONFIG_MANILA_NETWORK_STANDALONE_IP_RANGE=
CONFIG_MANILA_NETWORK_STANDALONE_IP_VERSION=4
CONFIG_MANILA_GLUSTERFS_SERVERS=
CONFIG_MANILA_GLUSTERFS_NATIVE_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_VOLUME_PATTERN=
CONFIG_MANILA_GLUSTERFS_TARGET=
CONFIG_MANILA_GLUSTERFS_MOUNT_POINT_BASE=
CONFIG_MANILA_GLUSTERFS_NFS_SERVER_TYPE=gluster
CONFIG_MANILA_GLUSTERFS_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_GANESHA_SERVER_IP=
CONFIG_HORIZON_SSL=n
CONFIG_HORIZON_SECRET_KEY=33cade531a764c858e4e6c22488f379f
CONFIG_HORIZON_SSL_CERT=
CONFIG_HORIZON_SSL_KEY=
CONFIG_HORIZON_SSL_CACERT=
CONFIG_SWIFT_KS_PW=PW_PLACEHOLDER
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=a940c9a54fbb4af8
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=36ce78ff06ef4577
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin
CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_IMAGE_NAME=cirros
CONFIG_PROVISION_IMAGE_URL=http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
CONFIG_PROVISION_IMAGE_FORMAT=qcow2
CONFIG_PROVISION_IMAGE_SSH_USER=cirros
CONFIG_TEMPEST_HOST=
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=PW_PLACEHOLDER
CONFIG_PROVISION_TEMPEST_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_RUN_TEMPEST=n
CONFIG_RUN_TEMPEST_TESTS=smoke
CONFIG_PROVISION_OVS_BRIDGE=n
CONFIG_CEILOMETER_SECRET=19ae0e7430174349
CONFIG_CEILOMETER_KS_PW=337b08d4b3a44753
CONFIG_CEILOMETER_COORDINATION_BACKEND=redis
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_REDIS_MASTER_HOST=192.169.142.127
CONFIG_REDIS_PORT=6379
CONFIG_REDIS_HA=n
CONFIG_REDIS_SLAVE_HOSTS=
CONFIG_REDIS_SENTINEL_HOSTS=
CONFIG_REDIS_SENTINEL_CONTACT_HOST=
CONFIG_REDIS_SENTINEL_PORT=26379
CONFIG_REDIS_SENTINEL_QUORUM=2
CONFIG_REDIS_MASTER_NAME=mymaster
CONFIG_AODH_KS_PW=acdd500a5fed4700
CONFIG_GNOCCHI_DB_PW=cf11b5d6205f40e7
CONFIG_GNOCCHI_KS_PW=36eba4690b224044
CONFIG_TROVE_DB_PW=PW_PLACEHOLDER
CONFIG_TROVE_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_NOVA_USER=trove
CONFIG_TROVE_NOVA_TENANT=services
CONFIG_TROVE_NOVA_PW=PW_PLACEHOLDER
CONFIG_SAHARA_DB_PW=PW_PLACEHOLDER
CONFIG_SAHARA_KS_PW=PW_PLACEHOLDER
CONFIG_NAGIOS_PW=02f168ee8edd44e4

Up on completion :-

[root@ServerCenttOS01 ~]# nova-manage version
13.0.0-0.20160304162843.c5a45a2.el7.centos

OVS external bridges activated on both  Network and Compute Node

***********************************************************
Upon completion on Network node 192.169.142.147
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.229"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

***********************************************************
On Network node 192.169.142.157
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.230"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

***********************************************************
On Compute node 192.169.142.137
***********************************************************
[root@ip-192-169-142-147 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="172.24.4.231"
NETMASK="255.255.255.240"
DNS1="83.221.202.254"
BROADCAST="172.24.4.239"
GATEWAY="172.24.4.225"
NM_CONTROLLED="no"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no

[root@ip-192-169-142-147 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
# HWADDR=00:22:15:63:E4:E2
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no


********************************************************************************
Next step was performed on both Network Nodes and Compute Node
********************************************************************************
#!/bin/bash -x
chkconfig network on
systemctl stop NetworkManager
systemctl disable NetworkManager
service network restart


At this point we start DVR/SNAT tuning



******************************************************************************************
On Controller an both Network Nodes  /etc/neutron/neutron.conf updated as follows
******************************************************************************************
dvr_base_mac = fa:16:3f:00:00:00

# System-wide flag to determine the type of router that tenants can create.
# Only admin can override. (boolean value)
router_distributed = True

# Enable HA mode for virtual routers. (boolean value)
l3_ha = True

******************************************************************************************
On  both Network Nodes  /etc/neutron/l3_agent.ini updated as follows
******************************************************************************************
[root@ServerCentOS02 neutron]# cat l3_agent.ini | grep -v ^# | grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
agent_mode = dvr_snat
external_network_bridge = br-ex
debug = False
[AGENT]


Files /etc/neutron/plugins/ml2/ml2_conf.ini,openvswitch_agent.ini,l3_agent.ini tuned on Network and Compute Nodes exactly as it was done in
RDO Liberty DVR Neutron workflow on CentOS 7.2
/etc/neutron/metadata_agent.ini copied over to Compute from Network Node.

**********************************************************************************
On Compute node neutron-l3-agent and neutron-metadata-agent are
supposed to be started via script
**********************************************************************************
 #!/bin/bash -x
 yum install  openstack-neutron-ml2  -y ;
 systemctl start neutron-l3-agent ;
 systemctl start neutron-metadata-agent ;
 systemctl restart neutron-openvswitch-agent ;
 systemctl enable neutron-l3-agent ;
 systemctl enable neutron-metadata-agent


All nodes have been rebooted.


Router was created via dashboard as RouterDVS :-

[root@ServerCenttOS01 ~(keystone_admin)]# neutron router-show RouterDVS
+-------------------------+------------------------------------------------------------------------+
| Field                   | Value                                                                  |
+-------------------------+------------------------------------------------------------------------+
| admin_state_up          | True                                                                   |
| availability_zone_hints |                                                                        |
| availability_zones      | nova                                                                   |
| distributed             | True                                                                   |
| external_gateway_info   | {"network_id": "1c347a42-21fa-4273-ad17-fa210d546ffd", "enable_snat":  |
|                         | true, "external_fixed_ips": [{"subnet_id": "fd24fa1d-cd2a-             |
|                         | 4a80-a822-e0a2fa5f743a", "ip_address": "172.24.4.227"}]}               |
| ha                      | True                                                                   |
| id                      | dd0d0741-c8a1-465a-8f89-ad986cd0592f                                   |
| name                    | RouterDVS                                                              |
| routes                  |                                                                        |
| status                  | ACTIVE                                                                 |
| tenant_id               | 06f56a00961e4c3ea10b537df8c86e1b                                       |
+-------------------------+------------------------------------------------------------------------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+------------------------------+----------------------------+----------------+-------+----------+
| id                           | host                       | admin_state_up | alive | ha_state |
+------------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-     | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                 |                            |                |       |          |
| c96930fa-066c-               | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf       |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-     | ServerCentOS03.localdomain | True           | :-)   | standby  |
| ef4ba2fe4105                 |                            |                |       |          |
+------------------------------+----------------------------+----------------+-------+----------+

Per https://review.openstack.org/#/c/196893/
Instead of running in the qrouter namespace, keepalived will run inside the snat-namespace. Therefore only snat ports will fall under the control of the HA domain.

  


[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ps -ef | grep keepalived
neutron   3168     1  0 00:29 ?        00:00:00 /usr/bin/python2 /bin/neutron-keepalived-state-change --router_id=dd0d0741-c8a1-465a-8f89-ad986cd0592f --namespace=snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f --conf_dir=/var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f --monitor_interface=ha-f3a6b78f-5f --monitor_cidr=169.254.0.1/24 --pid_file=/var/lib/neutron/external/pids/dd0d0741-c8a1-465a-8f89-ad986cd0592f.monitor.pid --state_path=/var/lib/neutron --user=988 --group=983
root      3385     1  0 00:29 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      3386  3385  0 00:29 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      7853  7677  0 00:56 pts/1    00:00:00 grep --color=auto keepalived

[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ip route
default via 172.24.4.225 dev qg-a31b1c28-8d
50.0.0.0/24 dev sg-3015f2cd-a4  proto kernel  scope link  src 50.0.0.11
169.254.0.0/24 dev ha-f3a6b78f-5f  proto kernel  scope link  src 169.254.0.1
169.254.192.0/18 dev ha-f3a6b78f-5f  proto kernel  scope link  src 169.254.192.1
172.24.4.224/28 dev qg-a31b1c28-8d  proto kernel  scope link  src 172.24.4.227

[root@ServerCentOS02 ~]# ssh 192.169.142.157
root@192.169.142.157's password:
Last login: Sun Mar 13 00:30:59 2016

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ps -ef | grep keepalived
neutron   2997     1  0 00:30 ?        00:00:00 /usr/bin/python2 /bin/neutron-keepalived-state-change --router_id=dd0d0741-c8a1-465a-8f89-ad986cd0592f --namespace=snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f --conf_dir=/var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f --monitor_interface=ha-f9dd88a2-33 --monitor_cidr=169.254.0.1/24 --pid_file=/var/lib/neutron/external/pids/dd0d0741-c8a1-465a-8f89-ad986cd0592f.monitor.pid --state_path=/var/lib/neutron --user=988 --group=983
root      3216     1  0 00:30 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      3217  3216  0 00:30 ?        00:00:00 keepalived -P -f /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f/keepalived.conf -p /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid -r /var/lib/neutron/ha_confs/dd0d0741-c8a1-465a-8f89-ad986cd0592f.pid-vrrp
root      7682  7614  0 00:58 pts/1    00:00:00 grep --color=auto keepalived

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f  ip route
169.254.192.0/18 dev ha-f9dd88a2-33  proto kernel  scope link  src 169.254.192.2

[root@ServerCentOS02 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f ifconfig
ha-f3a6b78f-5f: flags=4163  mtu 1450
        inet 169.254.192.1  netmask 255.255.192.0  broadcast 169.254.255.255

        inet6 fe80::f816:3eff:fec0:50ff  prefixlen 64  scopeid 0x20
        ether fa:16:3e:c0:50:ff  txqueuelen 0  (Ethernet)
        RX packets 8  bytes 684 (684.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1224  bytes 66336 (64.7 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-a31b1c28-8d: flags=4163  mtu 1450
        inet 172.24.4.227  netmask 255.255.255.240  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe4d:d973  prefixlen 64  scopeid 0x20
        ether fa:16:3e:4d:d9:73  txqueuelen 0  (Ethernet)
        RX packets 51  bytes 3981 (3.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 25  bytes 1910 (1.8 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

sg-3015f2cd-a4: flags=4163  mtu 1450
        inet 50.0.0.11  netmask 255.255.255.0  broadcast 0.0.0.0
        inet6 fe80::f816:3eff:fe8c:dbd3  prefixlen 64  scopeid 0x20
        ether fa:16:3e:8c:db:d3  txqueuelen 0  (Ethernet)
        RX packets 15  bytes 1282 (1.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 26  bytes 2020 (1.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

[root@ServerCentOS02 ~]# ssh 192.169.142.157
root@192.169.142.157's password:
Last login: Sun Mar 13 01:07:02 2016 from ip-192-169-142-147.ip.secureserver.net

[root@ServerCentOS03 ~]# ip netns exec snat-dd0d0741-c8a1-465a-8f89-ad986cd0592f  ifconfig
ha-f9dd88a2-33: flags=4163  mtu 1450
        inet 169.254.192.2  netmask 255.255.192.0  broadcast 169.254.255.255

        inet6 fe80::f816:3eff:fead:71  prefixlen 64  scopeid 0x20
        ether fa:16:3e:ad:00:71  txqueuelen 0  (Ethernet)
        RX packets 1215  bytes 65930 (64.3 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 11  bytes 954 (954.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

qg-a31b1c28-8d: flags=4163  mtu 1450
        ether fa:16:3e:4d:d9:73  txqueuelen 0  (Ethernet)
        RX packets 54  bytes 4270 (4.1 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

sg-3015f2cd-a4: flags=4163  mtu 1450
        ether fa:16:3e:8c:db:d3  txqueuelen 0  (Ethernet)
        RX packets 63  bytes 3922 (3.8 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1  bytes 110 (110.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

Verification is done. 


[root@ServerCenttOS01 ~(keystone_admin)]# neutron net-list
+-------------------------------+-------------------------------+-------------------------------+
| id                            | name                          | subnets                       |
+-------------------------------+-------------------------------+-------------------------------+
| 1c347a42-21fa-4273-ad17-fa210 | public                        | fd24fa1d-cd2a-                |
| d546ffd                         |                               | 4a80-a822-e0a2fa5f743a        |
|                                     |              | 172.24.4.224/28   |  <== External Network

| 498a3600-0b40-49c0-8ec1-c4b95 | private                       | 33478000-2584-4b24-8f39-1482c |
| 5a4335e                       |                               | 5b853af 10.0.0.0/24           |
| 70034a53-52c8-4665-9ed1-2dc7d | HA network tenant 06f56a00961 | c2bbd68c-0d9d-                |
| 3380a98                       | e4c3ea10b537df8c86e1b         | 49b1-a270-e98bdd08783e        |
|                               |                               | 169.254.192.0/18              |
| 08607e5c-fc14-488d-9c9c-      | demo_network                  | ebd72d77-6ea2-4d4e-           |
| 4d5e14040a6e                  |                               | a5e2-650e745d3db6 50.0.0.0/24 |
+-------------------------------+-------------------------------+-------------------------------

******************************************************************************
During run-time Network Nodes have been randomly shutdown
*******************************************************************************
[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   | <=== Brought down
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | standby   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | xxx   | standby  |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | active   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | standby  |  <== Brought up again
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | active   |
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+



[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | xxx   | standby  | <== Brought  down
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

[root@ServerCenttOS01 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDVS
+-----------------------------+----------------------------+----------------+-------+----------+
| id                          | host                       | admin_state_up | alive | ha_state |
+-----------------------------+----------------------------+----------------+-------+----------+
| f356bbd0-804d-4ec7-82db-    | ServerCentOS04.localdomain | True           | :-)   | standby  |
| 6e2de8914277                |                            |                |       |          |
| c96930fa-066c-              | ServerCentOS02.localdomain | True           | :-)   | active   |
| 40d6-8096-44476980cedf      |                            |                |       |          |
| 54c2a8f3-4c64-46a8-997b-    | ServerCentOS03.localdomain | True           | :-)   | standby  | <=== Brought up again
| ef4ba2fe4105                |                            |                |       |          |
+-----------------------------+----------------------------+----------------+-------+----------+

  
  
  

Friday, March 4, 2016

Setup DVR on RDO Liberty Controller && 2(x)Computes ML2/OVS/VLAN landscape

UPDATE 03/05/2016
1. Inroduced in Kilo   VLAN network type support for distributed virtual routers(DVR) 
2. DVR setup on top of RDO Kilo ML2/OVS/VLAN deployment on Fedora 23
follows same guide lines as on Liberty.
END UPDATE
  
Just a reminder in Juno and Kilo DVR was available for deployments using VXLAN tunneling and required l2population activation on all nodes. One of new features of Liberty is DVR compatibility with ML2&OVS&VLAN deployed landscapes. On RDO Liberty packstack doesn't play so nicely doing VLAN deployment as in case of  VXLAN tunneling. Attempt to use old templates for answer file just does all configs properly only on Controller/Network Node.
  However, it is not a problem replicate across Compute Nodes landscape required samples ifcfg-br-eth1,ifcfg-eth1 ( supporting VLAN vm/data network ) and  openvswitch_agent.ini , what makes RDO Liberty system attractive not only in case VXLAN (GRE) tunneling deployments, but still pretty comfortable for VLAN setups.  
    I also have to notice that on RDO Kilo same answer-file does Compute
Nodes automatically and properly .

DVR setup on VLAN landscape is just easier then in case with VXLAN tunneling, l2population bringing up is not required 

[root@ServerCentOS01 ~]# cat  answerVLAN.txt
[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_MANILA_INSTALL=n
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=y
CONFIG_CEILOMETER_INSTALL=y
CONFIG_SAHARA_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_TROVE_INSTALL=n
CONFIG_IRONIC_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=y
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137,192.169.142.147
CONFIG_NETWORK_HOSTS=192.169.142.127

CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_USE_SUBNETS=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAMES=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_SAHARA_HOST=192.169.142.127
CONFIG_USE_EPEL=n
CONFIG_REPO=
CONFIG_ENABLE_RDO_TESTING=n
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_SSL_CACERT_FILE=/etc/pki/tls/certs/selfcert.crt
CONFIG_SSL_CACERT_KEY_FILE=/etc/pki/tls/private/selfkey.key
CONFIG_SSL_CERT_DIR=~/packstackca/
CONFIG_SSL_CACERT_SELFSIGN=y
CONFIG_SELFSIGN_CACERT_SUBJECT_C=--
CONFIG_SELFSIGN_CACERT_SUBJECT_ST=State
CONFIG_SELFSIGN_CACERT_SUBJECT_L=City
CONFIG_SELFSIGN_CACERT_SUBJECT_O=openstack
CONFIG_SELFSIGN_CACERT_SUBJECT_OU=packstack
CONFIG_SELFSIGN_CACERT_SUBJECT_CN=ServerCentOS01.localdomain
CONFIG_SELFSIGN_CACERT_SUBJECT_MAIL=admin@ServerCentOS01.localdomain
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=66d38468ec974117
CONFIG_KEYSTONE_DB_PW=e678440a531c47fe
CONFIG_KEYSTONE_DB_PURGE_ENABLE=True
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=e1e05295c5554685b678c91ed83b10b1
CONFIG_KEYSTONE_ADMIN_EMAIL=root@localhost
CONFIG_KEYSTONE_ADMIN_USERNAME=admin
CONFIG_KEYSTONE_ADMIN_PW=89e47996c1c54577
CONFIG_KEYSTONE_DEMO_PW=22182b9b61fa4c89
CONFIG_KEYSTONE_API_VERSION=v2.0
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_KEYSTONE_IDENTITY_BACKEND=sql
CONFIG_KEYSTONE_LDAP_URL=ldap://192.169.142.127
CONFIG_KEYSTONE_LDAP_USER_DN=
CONFIG_KEYSTONE_LDAP_USER_PASSWORD=
CONFIG_KEYSTONE_LDAP_SUFFIX=
CONFIG_KEYSTONE_LDAP_QUERY_SCOPE=one
CONFIG_KEYSTONE_LDAP_PAGE_SIZE=-1
CONFIG_KEYSTONE_LDAP_USER_SUBTREE=
CONFIG_KEYSTONE_LDAP_USER_FILTER=
CONFIG_KEYSTONE_LDAP_USER_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_USER_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_MAIL_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_MASK=-1
CONFIG_KEYSTONE_LDAP_USER_ENABLED_DEFAULT=TRUE
CONFIG_KEYSTONE_LDAP_USER_ENABLED_INVERT=n
CONFIG_KEYSTONE_LDAP_USER_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_USER_DEFAULT_PROJECT_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_USER_PASS_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_EMULATION_DN=
CONFIG_KEYSTONE_LDAP_USER_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_GROUP_SUBTREE=
CONFIG_KEYSTONE_LDAP_GROUP_FILTER=
CONFIG_KEYSTONE_LDAP_GROUP_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_GROUP_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_MEMBER_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_DESC_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_GROUP_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_USE_TLS=n
CONFIG_KEYSTONE_LDAP_TLS_CACERTDIR=
CONFIG_KEYSTONE_LDAP_TLS_CACERTFILE=
CONFIG_KEYSTONE_LDAP_TLS_REQ_CERT=demand
CONFIG_GLANCE_DB_PW=e695f8551e3d434c
CONFIG_GLANCE_KS_PW=ebdba320a904449e
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=cfc234405cdf4181
CONFIG_CINDER_DB_PURGE_ENABLE=True
CONFIG_CINDER_KS_PW=ba8df890a6454866
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=2G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES=
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=/etc/cinder/shares.conf
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_PARTNER_BACKEND_NAME=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_ESERIES_HOST_TYPE=linux_dm_mp
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_MANILA_DB_PW=PW_PLACEHOLDER
CONFIG_MANILA_KS_PW=PW_PLACEHOLDER
CONFIG_MANILA_BACKEND=generic
CONFIG_MANILA_NETAPP_DRV_HANDLES_SHARE_SERVERS=false
CONFIG_MANILA_NETAPP_TRANSPORT_TYPE=https
CONFIG_MANILA_NETAPP_LOGIN=admin
CONFIG_MANILA_NETAPP_PASSWORD=
CONFIG_MANILA_NETAPP_SERVER_HOSTNAME=
CONFIG_MANILA_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_MANILA_NETAPP_SERVER_PORT=443
CONFIG_MANILA_NETAPP_AGGREGATE_NAME_SEARCH_PATTERN=(.*)
CONFIG_MANILA_NETAPP_ROOT_VOLUME_AGGREGATE=
CONFIG_MANILA_NETAPP_ROOT_VOLUME_NAME=root
CONFIG_MANILA_NETAPP_VSERVER=
CONFIG_MANILA_GENERIC_DRV_HANDLES_SHARE_SERVERS=true
CONFIG_MANILA_GENERIC_VOLUME_NAME_TEMPLATE=manila-share-%s
CONFIG_MANILA_GENERIC_SHARE_MOUNT_PATH=/shares
CONFIG_MANILA_SERVICE_IMAGE_LOCATION=https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2
CONFIG_MANILA_SERVICE_INSTANCE_USER=ubuntu
CONFIG_MANILA_SERVICE_INSTANCE_PASSWORD=ubuntu
CONFIG_MANILA_NETWORK_TYPE=neutron
CONFIG_MANILA_NETWORK_STANDALONE_GATEWAY=
CONFIG_MANILA_NETWORK_STANDALONE_NETMASK=
CONFIG_MANILA_NETWORK_STANDALONE_SEG_ID=
CONFIG_MANILA_NETWORK_STANDALONE_IP_RANGE=
CONFIG_MANILA_NETWORK_STANDALONE_IP_VERSION=4
CONFIG_MANILA_GLUSTERFS_SERVERS=
CONFIG_MANILA_GLUSTERFS_NATIVE_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_VOLUME_PATTERN=
CONFIG_MANILA_GLUSTERFS_TARGET=
CONFIG_MANILA_GLUSTERFS_MOUNT_POINT_BASE=
CONFIG_MANILA_GLUSTERFS_NFS_SERVER_TYPE=gluster
CONFIG_MANILA_GLUSTERFS_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_GANESHA_SERVER_IP=
CONFIG_IRONIC_DB_PW=PW_PLACEHOLDER
CONFIG_IRONIC_KS_PW=PW_PLACEHOLDER
CONFIG_NOVA_DB_PURGE_ENABLE=True
CONFIG_NOVA_DB_PW=48db80033113424c
CONFIG_NOVA_KS_PW=04750d997f7b4cd0
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_MANAGER=nova.compute.manager.ComputeManager
CONFIG_VNC_SSL_CERT=
CONFIG_VNC_SSL_KEY=
CONFIG_NOVA_PCI_ALIAS=
CONFIG_NOVA_PCI_PASSTHROUGH_WHITELIST=
CONFIG_NOVA_COMPUTE_PRIVIF=eth1
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=eth1
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=3549700b2072492f
CONFIG_NEUTRON_DB_PW=2896cb32038040d1
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_METADATA_PW=8a5ceeb9f9264e7c
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_VPNAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vlan,flat
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=physnet1:100:200

CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=
CONFIG_NEUTRON_ML2_VXLAN_GROUP=
CONFIG_NEUTRON_ML2_VNI_RANGES=
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_ML2_SUPPORTED_PCI_VENDOR_DEVS=['15b3:1004', '8086:10ca']
CONFIG_NEUTRON_ML2_SRIOV_AGENT_REQUIRED=n
CONFIG_NEUTRON_ML2_SRIOV_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-eth1
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=br-eth1:eth1

CONFIG_NEUTRON_OVS_TUNNEL_IF=
# CONFIG_NEUTRON_OVS_TUNNEL_SUBNETS=12.0.0.0/24
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_HORIZON_SSL=n
CONFIG_HORIZON_SECRET_KEY=70c8f53f8a1948768d432c11bbffe94a
CONFIG_HORIZON_SSL_CERT=
CONFIG_HORIZON_SSL_KEY=
CONFIG_HORIZON_SSL_CACERT=
CONFIG_SWIFT_KS_PW=2ad1d724dc6d4764
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=038b40edd19f4251
CONFIG_SWIFT_STORAGE_SIZE=20G

CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=c863b530aaa24240
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin
CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_IMAGE_NAME=cirros
CONFIG_PROVISION_IMAGE_URL=http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
CONFIG_PROVISION_IMAGE_FORMAT=qcow2
CONFIG_PROVISION_IMAGE_SSH_USER=cirros
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=PW_PLACEHOLDER
CONFIG_PROVISION_TEMPEST_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_OVS_BRIDGE=y
CONFIG_CEILOMETER_SECRET=8b362a1d225a472d
CONFIG_CEILOMETER_KS_PW=92bb4ec7c7584e18
CONFIG_CEILOMETER_COORDINATION_BACKEND=redis
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_REDIS_MASTER_HOST=192.169.142.127
CONFIG_REDIS_PORT=6379
CONFIG_REDIS_HA=n
CONFIG_REDIS_SLAVE_HOSTS=
CONFIG_REDIS_SENTINEL_HOSTS=
CONFIG_REDIS_SENTINEL_CONTACT_HOST=
CONFIG_REDIS_SENTINEL_PORT=26379
CONFIG_REDIS_SENTINEL_QUORUM=2
CONFIG_REDIS_MASTER_NAME=mymaster
CONFIG_SAHARA_DB_PW=PW_PLACEHOLDER
CONFIG_SAHARA_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_DB_PW=PW_PLACEHOLDER
CONFIG_TROVE_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_NOVA_USER=trove
CONFIG_TROVE_NOVA_TENANT=services
CONFIG_TROVE_NOVA_PW=PW_PLACEHOLDER
CONFIG_NAGIOS_PW=c5b1af910c094f3a

Controller would be done OK. Files generated may serve as samples
for Compute Nodes

******************************************************************************************
Compute Node requires manual updates. Copy over from Controller ifcfg-br-eth1 OVS bridge and ifcfg-eth1 OVS port , /etc/neutron/plugins/ml2neutron_ovsagent.ini,/etc/neutron/plugins/ml2/ml2_conf.ini to Compute Nodes
******************************************************************************************
Create symlink pluging.ini under /etc/neutron

[root@ServerCentOS02 neutron(keystone_admin)]# ls -l
total 80
drwxr-xr-x. 10 root root     4096 Mar  1 22:12 conf.d
-rw-r-----.  1 root neutron  4476 Dec  7 18:53 dhcp_agent.ini
-rw-r-----.  1 root neutron  5537 Dec  7 18:53 l3_agent.ini
-rw-r-----.  1 root neutron  2600 Dec  7 18:53 metadata_agent.ini
-rw-r-----.  1 root neutron 37034 Mar  1 22:12 neutron.conf
lrwxrwxrwx.  1 root root       37 Mar  1 22:38 plugin.ini -&gt; /etc/neutron/plugins/ml2/ml2_conf.ini
drwxr-xr-x.  3 root root       16 Mar  1 22:12 plugins
-rw-r-----.  1 root neutron  9486 Dec  7 18:53 policy.json
-rw-r--r--.  1 root root     1195 Dec  7 18:53 rootwrap.conf

[root@ServerCentOS02 neutron(keystone_admin)]# cat plugin.ini | grep -v ^#|grep -v ^$
[ml2]
type_drivers = vlan,flat
tenant_network_types = vlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
flat_networks =*
[ml2_type_vlan]
network_vlan_ranges =physnet1:100:200
[ml2_type_gre]
[ml2_type_vxlan]
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

***********************************************************************************
Update local_ip in  openvswitch_agent.ini corresondently on 192.169.142.137
and  192.169.142.147
***********************************************************************************

[root@ServerCentOS02 ml2(keystone_admin)]# cat openvswitch_agent.ini | grep -v ^# | grep -v ^$
[ovs]

network_vlan_ranges = physnet1:100:200
tenant_network_type = vlan
enable_tunneling = False
integration_bridge = br-int
bridge_mappings = physnet1:br-eth1
local_ip = 192.168.1.137
[agent]
polling_interval = 2
l2_population = False
arp_responder = False
prevent_arp_spoofing = True
enable_distributed_routing = False
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

*******************************************************************
Copy over from Controller following ifcfg-* files ,
support VLAN tagged vm/data network connection from
Compute to Controller and vice/versa
*******************************************************************

[root@ServerCentOS02 network-scripts(keystone_admin)]# cat ifcfg-br-eth1
ONBOOT=yes
NM_CONTROLLED=no
DEVICE=br-eth1
DEVICETYPE=ovs
OVSBOOTPROTO=none
TYPE=OVSBridge

[root@ServerCentOS02 network-scripts(keystone_admin)]# cat ifcfg-eth1
DEVICE=eth1
DEVICETYPE=ovs
TYPE=OVSPort
OVS_BRIDGE=br-eth1
ONBOOT=yes
BOOTPROTO=none

************************************
When done on each Compute Node
************************************
Run script

#!/bin/bash -x
chkconfig network on ;
systemctl stop NetworkManager ;
systemctl disable NetworkManager ;
service network restart
 
Followed by `openstack-service restart`  and Node reboot.
Afterwards `ovs-vsctl show` would look like ( on Compute )

[root@ServerCentOS02 ~(keystone_admin)]# ovs-vsctl show
58d168ad-6076-409d-8d61-0f2ca1481da8
    Bridge "br-eth1"
        Port "eth1"
            Interface "eth1"
        Port "br-eth1"
            Interface "br-eth1"
                type: internal
        Port "phy-br-eth1"
            Interface "phy-br-eth1"
                type: patch
                options: {peer="int-br-eth1"}
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port "int-br-eth1"
            Interface "int-br-eth1"
                type: patch
                options: {peer="phy-br-eth1"}
        Port "qvode5c08bd-cf"
            tag: 1
            Interface "qvode5c08bd-cf"
    ovs_version: "2.4.0"

**********************************************************
Controller status right after packstack completion
**********************************************************

[root@ServerCentOS01 neutron(keystone_admin)]# ls -l
total 84
-rw-r-----.  1 root root      182 Mar  1 22:12 api-paste.ini
drwxr-xr-x. 10 root root     4096 Mar  1 22:12 conf.d
-rw-r-----.  1 root neutron  4867 Mar  1 22:12 dhcp_agent.ini
-rw-r-----.  1 root neutron  5856 Mar  1 22:12 l3_agent.ini
-rw-r-----.  1 root neutron  2838 Mar  1 22:12 metadata_agent.ini
-rw-r-----.  1 root neutron 37747 Mar  1 22:12 neutron.conf
lrwxrwxrwx.  1 root root       37 Mar  1 22:12 plugin.ini -> /etc/neutron/plugins/ml2/ml2_conf.ini
drwxr-xr-x.  3 root root       16 Mar  1 22:12 plugins
-rw-r-----.  1 root neutron  9486 Dec  7 18:53 policy.json
-rw-r--r--.  1 root root     1195 Dec  7 18:53 rootwrap.conf

[root@ServerCentOS01 neutron(keystone_admin)]# cat plugin.ini | grep -v ^$|grep -v ^#
[ml2]
type_drivers = vlan,flat
tenant_network_types = vlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
flat_networks =*
[ml2_type_vlan]
network_vlan_ranges =physnet1:100:200
[ml2_type_gre]
[ml2_type_vxlan]
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

[root@ServerCentOS01 neutron(keystone_admin)]# cd plugins/ml2
[root@ServerCentOS01 ml2(keystone_admin)]# pwd
/etc/neutron/plugins/ml2
[root@ServerCentOS01 ml2(keystone_admin)]# cat openvswitch_agent.ini | grep -v ^$|grep -v ^#
[ovs]
integration_bridge = br-int
local_ip = 192.169.142.127
bridge_mappings =physnet1:br-eth1
enable_tunneling=False
[agent]
polling_interval = 2
l2_population = False
arp_responder = False
prevent_arp_spoofing = True
enable_distributed_routing = False
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

[root@ServerCentOS01 ~]# ovs-vsctl show
5f4d0aa9-049c-4522-a6bf-4c10ba12dcc2
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth0"
            Interface "eth0"
        Port "qg-a1a797a9-9b"
            Interface "qg-a1a797a9-9b"
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "tap35d4e073-fd"
            tag: 1
            Interface "tap35d4e073-fd"
                type: internal
        Port "tap257a060c-22"
            tag: 2
            Interface "tap257a060c-22"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qr-71a634e2-2b"
            tag: 2
            Interface "qr-71a634e2-2b"
                type: internal
        Port "int-br-eth1"
            Interface "int-br-eth1"
                type: patch
                options: {peer="phy-br-eth1"}
    Bridge "br-eth1"
        Port "br-eth1"
            Interface "br-eth1"
                type: internal
        Port "phy-br-eth1"
            Interface "phy-br-eth1"
                type: patch
                options: {peer="int-br-eth1"}
        Port "eth1"
            Interface "eth1"
    ovs_version: "2.4.0"

*******************
DVR  SETUP
*******************

On Controller (X=2) and Computes X=(3,4) update :-

# cat ifcfg-br-ex
DEVICE="br-ex"
BOOTPROTO="static"
IPADDR="192.169.142.1(X)7"
NETMASK="255.255.255.0"
DNS1="83.221.202.254"
BROADCAST="192.169.142.255"
GATEWAY="192.169.142.1"
NM_CONTROLLED="no"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="yes"
IPV6INIT=no
ONBOOT="yes"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex

DEVICETYPE="ovs"

# cat ifcfg-eth0
DEVICE="eth0"
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no


Followed by `service network restart`

*****************************************
On Controller update neutron.conf
*****************************************
router_distributed = True
dvr_base_mac = fa:16:3f:00:00:00

 [root@ip-192-169-142-127 neutron(keystone_admin)]# cat l3_agent.ini | grep -v ^#| grep -v ^$
[DEFAULT]
debug = False
interface_driver =neutron.agent.linux.interface.OVSInterfaceDriver
handle_internal_only_routers = True
external_network_bridge = br-ex
metadata_port = 9697
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5
enable_metadata_proxy = True
router_delete_namespaces = False
agent_mode = dvr_snat
[AGENT]

Followed by `openstack-service restart` and status verification

*********************************
On each Compute Node
*********************************

[root@ip-192-169-142-147 neutron]# cat l3_agent.ini | grep -v ^#| grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
agent_mode = dvr
[AGENT]


Replicate metadata_agent.ini from Controller/Network Node to
all Compute Nodes on your landscape

************************************************************************************
plugin.ini which is symlink to /etc/neutron/plugins/ml2/ml2_conf.ini
stays the same
************************************************************************************

[root@ServerCentOS02 neutron(keystone_admin)]# cat plugin.ini | grep -v ^#|grep -v ^$
[ml2]
type_drivers = vlan,flat
tenant_network_types = vlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
flat_networks =*
[ml2_type_vlan]
network_vlan_ranges =physnet1:100:200
[ml2_type_gre]
[ml2_type_vxlan]
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

**********************************************
One update to openvswitch_agent.ini
**********************************************
[root@ServerCentOS02 ml2(keystone_admin)]# cat openvswitch_agent.ini | grep -v ^# | grep -v ^$
[ovs]

network_vlan_ranges = physnet1:100:200
tenant_network_type = vlan
enable_tunneling = False
integration_bridge = br-int
bridge_mappings = physnet1:br-eth1
local_ip = 192.168.1.137
[agent]
polling_interval = 2
l2_population = False
arp_responder = False
prevent_arp_spoofing = True
enable_distributed_routing =True  <== here
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

**********************************************************************************
On each Compute node neutron-l3-agent and neutron-metadata-agent are
supposed to be started via script
**********************************************************************************
 #!/bin/bash -x
 yum install  openstack-neutron-ml2  -y ;
 systemctl start neutron-l3-agent ;
 systemctl start neutron-metadata-agent ;
 systemctl restart neutron-openvswitch-agent ;
 systemctl enable neutron-l3-agent ;
 systemctl enable neutron-metadata-agent


****************************************************************
Followed by `openstack-service restart` and status verification
for each Compute Node
****************************************************************
[root@ServerCentOS02 ~]# openstack-status
== Nova services ==
openstack-nova-api:                     inactive  (disabled on boot)
openstack-nova-compute:                 active
openstack-nova-network:                 inactive  (disabled on boot)
openstack-nova-scheduler:               inactive  (disabled on boot)
== neutron services ==
neutron-server:                         inactive  (disabled on boot)
neutron-dhcp-agent:                     inactive  (disabled on boot)
neutron-l3-agent:                       active
neutron-metadata-agent:                 active
neutron-openvswitch-agent:              active

== Ceilometer services ==
openstack-ceilometer-api:               inactive  (disabled on boot)
openstack-ceilometer-central:           inactive  (disabled on boot)
openstack-ceilometer-compute:           active
openstack-ceilometer-collector:         inactive  (disabled on boot)
== Support services ==
openvswitch:                            active
dbus:                                   active
Warning novarc not sourced

[root@ServerCentOS02 ~]# ovs-vsctl show
58d168ad-6076-409d-8d61-0f2ca1481da8
    Bridge "br-eth1"
        Port "phy-br-eth1"
            Interface "phy-br-eth1"
                type: patch
                options: {peer="int-br-eth1"}
        Port "br-eth1"
            Interface "br-eth1"
                type: internal
        Port "eth1"
            Interface "eth1"
    Bridge br-ex
        Port "eth0"
            Interface "eth0"
        Port br-ex
            Interface br-ex
                type: internal

        Port "fg-3400fce2-f3"
            Interface "fg-3400fce2-f3"
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qvo860a1d40-2c"
            tag: 1
            Interface "qvo860a1d40-2c"
        Port br-int
            Interface br-int
                type: internal
        Port "qr-95a16271-1a"
            tag: 1
            Interface "qr-95a16271-1a"
                type: internal
        Port "int-br-eth1"
            Interface "int-br-eth1"
                type: patch
                options: {peer="phy-br-eth1"}
        Port "qvof1a54ff1-9b"
            tag: 1
            Interface "qvof1a54ff1-9b"
    ovs_version: "2.4.0"

   
  
*********************************************
Neutron work flow Controller
*********************************************
   [root@ServerCentOS01 ~(keystone_admin)]#  ovs-ofctl dump-flows br-eth1| grep NORMAL
 cookie=0x0, duration=9250.016s, table=0, n_packets=5, n_bytes=260, idle_age=9241, priority=0 actions=NORMAL
 cookie=0x0, duration=9239.547s, table=2, n_packets=3, n_bytes=126, idle_age=9234, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:106,NORMAL
 cookie=0x0, duration=9239.523s, table=2, n_packets=0, n_bytes=0, idle_age=9239, priority=4,in_port=2,dl_vlan=2 actions=mod_vlan_vid:182,NORMAL
 cookie=0x0, duration=9239.432s, table=2, n_packets=76, n_bytes=16883, idle_age=1825, priority=4,in_port=2,dl_vlan=3 actions=mod_vlan_vid:180,NORMAL

 cookie=0x0, duration=9240.182s, table=3, n_packets=4783, n_bytes=259509, idle_age=1, priority=1 actions=NORMAL

[root@ServerCentOS01 ~(keystone_admin)]#  ovs-ofctl dump-flows br-int| grep NORMAL
 cookie=0xa5bb54579234857d, duration=9258.450s, table=0, n_packets=0, n_bytes=0, idle_age=9258, priority=3,in_port=1,dl_vlan=106 actions=mod_vlan_vid:1,NORMAL
 cookie=0xa5bb54579234857d, duration=9258.407s, table=0, n_packets=0, n_bytes=0, idle_age=9258, priority=3,in_port=1,dl_vlan=182 actions=mod_vlan_vid:2,NORMAL
 cookie=0xa5bb54579234857d, duration=9258.333s, table=0, n_packets=111, n_bytes=13103, idle_age=1840, priority=3,in_port=1,dl_vlan=180 actions=mod_vlan_vid:3,NORMAL

 cookie=0xa5bb54579234857d, duration=9269.283s, table=0, n_packets=0, n_bytes=0, idle_age=9269, priority=0 actions=NORMAL
 cookie=0xa5bb54579234857d, duration=9259.178s, table=0, n_packets=101, n_bytes=18909, idle_age=1844, priority=1 actions=NORMAL

Compare numbers on br-eth1 ,connecting Compute and Controller, and external bridge br-ex local to particular Compute Node


*********************************************
Neutron work flow Compute 1
*********************************************

[root@ServerCentOS01 ~(keystone_admin)]# ssh 192.169.142.137
Last login: Fri Mar  4 15:30:56 2016 from ip-192-169-142-127.ip.secureserver.net

[root@ServerCentOS02 ~]#  ovs-ofctl dump-flows br-eth1 | grep NORMAL
 cookie=0x0, duration=9333.056s, table=0, n_packets=0, n_bytes=0, idle_age=9333, priority=0 actions=NORMAL
 cookie=0x0, duration=8878.554s, table=2, n_packets=112, n_bytes=12791, idle_age=1858, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:180,NORMAL
 cookie=0x0, duration=9332.902s, table=3, n_packets=4770, n_bytes=256299, idle_age=0, priority=1 actions=NORMAL

[root@ServerCentOS02 ~]#  ovs-ofctl dump-flows br-int | grep NORMAL
 cookie=0xb29237804b1bf15f, duration=8886.372s, table=0, n_packets=46, n_bytes=7219, idle_age=1870, priority=3,in_port=1,dl_vlan=180 actions=mod_vlan_vid:1,NORMAL
 cookie=0xb29237804b1bf15f, duration=9340.926s, table=0, n_packets=0, n_bytes=0, idle_age=9340, priority=0 actions=NORMAL
 cookie=0xb29237804b1bf15f, duration=9340.746s, table=0, n_packets=4906525, n_bytes=4998841358, idle_age=0, priority=1 actions=NORMAL
 cookie=0xb29237804b1bf15f, duration=8886.340s, table=24, n_packets=0, n_bytes=0, idle_age=8886, priority=2,icmp6,in_port=3,icmp_type=136,nd_target=fe80::f816:3eff:fe77:fd3c actions=NORMAL
 cookie=0xb29237804b1bf15f, duration=7998.122s, table=24, n_packets=0, n_bytes=0, idle_age=7998, priority=2,icmp6,in_port=5,icmp_type=136,nd_target=fe80::f816:3eff:fe91:1446 actions=NORMAL
 cookie=0xb29237804b1bf15f, duration=8886.323s, table=24, n_packets=3, n_bytes=126, idle_age=8875, priority=2,arp,in_port=3,arp_spa=70.0.0.15 actions=NORMAL
 cookie=0xb29237804b1bf15f, duration=7998.113s, table=24, n_packets=53, n_bytes=2226, idle_age=3, priority=2,arp,in_port=5,arp_spa=70.0.0.17 actions=NORMAL

[root@ServerCentOS02 ~]#  ovs-ofctl dump-flows br-ex
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=12992.545s, table=0, n_packets=6096113, n_bytes=6160846107, idle_age=0, priority=0 actions=NORMAL

*********************************************
Neutron work flow Compute 2
*********************************************
[root@ServerCentOS02 ~]# ssh 192.169.142.147
root@192.169.142.147's password:
Last login: Fri Mar  4 15:31:43 2016 from ip-192-169-142-137.ip.secureserver.net
[root@ServerCentOS03 ~]# ovs-ofctl dump-flows br-eth1 | grep NORMAL
 cookie=0x0, duration=9390.453s, table=0, n_packets=0, n_bytes=0, idle_age=9390, priority=0 actions=NORMAL
 cookie=0x0, duration=8607.979s, table=2, n_packets=68, n_bytes=7003, idle_age=8525, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:180,NORMAL
 cookie=0x0, duration=9390.341s, table=3, n_packets=4884, n_bytes=273145, idle_age=1, priority=1 actions=NORMAL

[root@ServerCentOS03 ~]# ovs-ofctl dump-flows br-int | grep NORMAL
 cookie=0xb74d262de8ca4f34, duration=8621.337s, table=0, n_packets=110, n_bytes=20309, idle_age=1900, priority=3,in_port=1,dl_vlan=180 actions=mod_vlan_vid:1,NORMAL
 cookie=0xb74d262de8ca4f34, duration=9403.851s, table=0, n_packets=0, n_bytes=0, idle_age=9403, priority=0 actions=NORMAL
 cookie=0xb74d262de8ca4f34, duration=9403.721s, table=0, n_packets=12358647, n_bytes=12580495273, idle_age=408, priority=1 actions=NORMAL
 cookie=0xb74d262de8ca4f34, duration=8621.313s, table=24, n_packets=0, n_bytes=0, idle_age=8621, priority=2,icmp6,in_port=3,icmp_type=136,nd_target=fe80::f816:3eff:fefe:8274 actions=NORMAL
 cookie=0xb74d262de8ca4f34, duration=8621.307s, table=24, n_packets=11, n_bytes=462, idle_age=408, priority=2,arp,in_port=3,arp_spa=70.0.0.16 actions=NORMAL

[root@ServerCentOS03 ~]# ovs-ofctl dump-flows br-ex
NXST_FLOW reply (xid=0x4):
 cookie=0x0, duration=12712.519s, table=0, n_packets=12685802, n_bytes=12627356349, idle_age=0, priority=0 actions=NORMAL