Openstack RDO && KVM Hypervisor: December 2015

Friday, December 25, 2015

DVR with Two external networks via flat network provider on CentOS 7.2 RDO Liberty

Post is actually step by step procedure of creating DVR system working with two external networks been built via flat network provider. Question which several times was raised up at ask.openstack.org, however was not addressed properly.

************************************************************************************
1. Setup Controller/Network + Compute ML2&OVS&VXLAN via answer-file
************************************************************************************

[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_MANILA_INSTALL=n
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=y
CONFIG_CEILOMETER_INSTALL=y
CONFIG_SAHARA_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_TROVE_INSTALL=n
CONFIG_IRONIC_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=n
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137
CONFIG_NETWORK_HOSTS=192.169.142.127
CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_USE_SUBNETS=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAME=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_SAHARA_HOST=192.169.142.127
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_ENABLE_RDO_TESTING=n
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_SSL_CACERT_FILE=/etc/pki/tls/certs/selfcert.crt
CONFIG_SSL_CACERT_KEY_FILE=/etc/pki/tls/private/selfkey.key
CONFIG_SSL_CERT_DIR=~/packstackca/
CONFIG_SSL_CACERT_SELFSIGN=y
CONFIG_SELFSIGN_CACERT_SUBJECT_C=--
CONFIG_SELFSIGN_CACERT_SUBJECT_ST=State
CONFIG_SELFSIGN_CACERT_SUBJECT_L=City
CONFIG_SELFSIGN_CACERT_SUBJECT_O=openstack
CONFIG_SELFSIGN_CACERT_SUBJECT_OU=packstack
CONFIG_SELFSIGN_CACERT_SUBJECT_CN=ip-192-169-142-127.ip.secureserver.net
CONFIG_SELFSIGN_CACERT_SUBJECT_MAIL=admin@ip-192-169-142-127.ip.secureserver.net
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=7207ae344ed04957
CONFIG_KEYSTONE_DB_PW=abcae16b785245c3
CONFIG_KEYSTONE_DB_PURGE_ENABLE=True
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=3ad2de159f9649afb0c342ba57e637d9
CONFIG_KEYSTONE_ADMIN_EMAIL=root@localhost
CONFIG_KEYSTONE_ADMIN_USERNAME=admin
CONFIG_KEYSTONE_ADMIN_PW=7049f834927e4468
CONFIG_KEYSTONE_DEMO_PW=bf737b785cfa4398
CONFIG_KEYSTONE_API_VERSION=v2.0
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_KEYSTONE_IDENTITY_BACKEND=sql
CONFIG_KEYSTONE_LDAP_URL=ldap://192.169.142.127
CONFIG_KEYSTONE_LDAP_USER_DN=
CONFIG_KEYSTONE_LDAP_USER_PASSWORD=
CONFIG_KEYSTONE_LDAP_SUFFIX=
CONFIG_KEYSTONE_LDAP_QUERY_SCOPE=one
CONFIG_KEYSTONE_LDAP_PAGE_SIZE=-1
CONFIG_KEYSTONE_LDAP_USER_SUBTREE=
CONFIG_KEYSTONE_LDAP_USER_FILTER=
CONFIG_KEYSTONE_LDAP_USER_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_USER_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_MAIL_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_MASK=-1
CONFIG_KEYSTONE_LDAP_USER_ENABLED_DEFAULT=TRUE
CONFIG_KEYSTONE_LDAP_USER_ENABLED_INVERT=n
CONFIG_KEYSTONE_LDAP_USER_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_USER_DEFAULT_PROJECT_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_USER_PASS_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_EMULATION_DN=
CONFIG_KEYSTONE_LDAP_USER_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_GROUP_SUBTREE=
CONFIG_KEYSTONE_LDAP_GROUP_FILTER=
CONFIG_KEYSTONE_LDAP_GROUP_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_GROUP_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_MEMBER_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_DESC_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_GROUP_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_USE_TLS=n
CONFIG_KEYSTONE_LDAP_TLS_CACERTDIR=
CONFIG_KEYSTONE_LDAP_TLS_CACERTFILE=
CONFIG_KEYSTONE_LDAP_TLS_REQ_CERT=demand
CONFIG_GLANCE_DB_PW=41264fc52ffd4fe8
CONFIG_GLANCE_KS_PW=f6a9398960534797
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=5ac08c6d09ba4b69
CONFIG_CINDER_DB_PURGE_ENABLE=True
CONFIG_CINDER_KS_PW=c8cb1ecb8c2b4f6f
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=5G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES=
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=/etc/cinder/shares.conf
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_PARTNER_BACKEND_NAME=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_ESERIES_HOST_TYPE=linux_dm_mp
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_MANILA_DB_PW=PW_PLACEHOLDER
CONFIG_MANILA_KS_PW=PW_PLACEHOLDER
CONFIG_MANILA_BACKEND=generic
CONFIG_MANILA_NETAPP_DRV_HANDLES_SHARE_SERVERS=false
CONFIG_MANILA_NETAPP_TRANSPORT_TYPE=https
CONFIG_MANILA_NETAPP_LOGIN=admin
CONFIG_MANILA_NETAPP_PASSWORD=
CONFIG_MANILA_NETAPP_SERVER_HOSTNAME=
CONFIG_MANILA_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_MANILA_NETAPP_SERVER_PORT=443
CONFIG_MANILA_NETAPP_AGGREGATE_NAME_SEARCH_PATTERN=(.*)
CONFIG_MANILA_NETAPP_ROOT_VOLUME_AGGREGATE=
CONFIG_MANILA_NETAPP_ROOT_VOLUME_NAME=root
CONFIG_MANILA_NETAPP_VSERVER=
CONFIG_MANILA_GENERIC_DRV_HANDLES_SHARE_SERVERS=true
CONFIG_MANILA_GENERIC_VOLUME_NAME_TEMPLATE=manila-share-%s
CONFIG_MANILA_GENERIC_SHARE_MOUNT_PATH=/shares
CONFIG_MANILA_SERVICE_IMAGE_LOCATION=https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2
CONFIG_MANILA_SERVICE_INSTANCE_USER=ubuntu
CONFIG_MANILA_SERVICE_INSTANCE_PASSWORD=ubuntu
CONFIG_MANILA_NETWORK_TYPE=neutron
CONFIG_MANILA_NETWORK_STANDALONE_GATEWAY=
CONFIG_MANILA_NETWORK_STANDALONE_NETMASK=
CONFIG_MANILA_NETWORK_STANDALONE_SEG_ID=
CONFIG_MANILA_NETWORK_STANDALONE_IP_RANGE=
CONFIG_MANILA_NETWORK_STANDALONE_IP_VERSION=4
CONFIG_MANILA_GLUSTERFS_SERVERS=
CONFIG_MANILA_GLUSTERFS_NATIVE_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_VOLUME_PATTERN=
CONFIG_MANILA_GLUSTERFS_TARGET=
CONFIG_MANILA_GLUSTERFS_MOUNT_POINT_BASE=
CONFIG_MANILA_GLUSTERFS_NFS_SERVER_TYPE=gluster
CONFIG_MANILA_GLUSTERFS_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_GANESHA_SERVER_IP=
CONFIG_IRONIC_DB_PW=PW_PLACEHOLDER
CONFIG_IRONIC_KS_PW=PW_PLACEHOLDER
CONFIG_NOVA_DB_PURGE_ENABLE=True
CONFIG_NOVA_DB_PW=1e1b5aeeeaf342a8
CONFIG_NOVA_KS_PW=d9583177a2444f06
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_MANAGER=nova.compute.manager.ComputeManager
CONFIG_VNC_SSL_CERT=
CONFIG_VNC_SSL_KEY=
CONFIG_NOVA_COMPUTE_PRIVIF=
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=808e36e154bd4cee
CONFIG_NEUTRON_DB_PW=0e2b927a21b44737
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_METADATA_PW=a965cd23ed2f4502
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_VPNAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan,flat
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_HORIZON_SSL=n
CONFIG_HORIZON_SECRET_KEY=a25b5ece9db24e2aba8d3a2b4d908ca5
CONFIG_HORIZON_SSL_CERT=
CONFIG_HORIZON_SSL_KEY=
CONFIG_HORIZON_SSL_CACERT=
CONFIG_SWIFT_KS_PW=8f75bfd461234c30
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=a60aacbedde7429a
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=976496a551b94296
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin
CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_IMAGE_NAME=cirros
CONFIG_PROVISION_IMAGE_URL=http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
CONFIG_PROVISION_IMAGE_FORMAT=qcow2
CONFIG_PROVISION_IMAGE_SSH_USER=cirros
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=PW_PLACEHOLDER
CONFIG_PROVISION_TEMPEST_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_OVS_BRIDGE=n
CONFIG_CEILOMETER_SECRET=19ae0e7430174349
CONFIG_CEILOMETER_KS_PW=337b08d4b3a44753
CONFIG_CEILOMETER_COORDINATION_BACKEND=redis
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_REDIS_MASTER_HOST=192.169.142.127
CONFIG_REDIS_PORT=6379
CONFIG_REDIS_HA=n
CONFIG_REDIS_SLAVE_HOSTS=
CONFIG_REDIS_SENTINEL_HOSTS=
CONFIG_REDIS_SENTINEL_CONTACT_HOST=
CONFIG_REDIS_SENTINEL_PORT=26379
CONFIG_REDIS_SENTINEL_QUORUM=2
CONFIG_REDIS_MASTER_NAME=mymaster
CONFIG_SAHARA_DB_PW=PW_PLACEHOLDER
CONFIG_SAHARA_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_DB_PW=PW_PLACEHOLDER
CONFIG_TROVE_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_NOVA_USER=trove
CONFIG_TROVE_NOVA_TENANT=services
CONFIG_TROVE_NOVA_PW=PW_PLACEHOLDER
CONFIG_NAGIOS_PW=PW_PLACEHOLDER

************************************************************************************
Four VNICs on each node MGMT (eth0) , VTEPS (eth1), EXT
Interfaces (eth2,eth3 )
************************************************************************************
Interfaces eth2,eth3 are attached to VMs via libvirt subnets :-

[root@fedora23wks ~]# cat external1.xml
<network>
   <name>external1</name>
   <uuid>d0a7964b-f93d-40c2-b749-b609aed52cf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr4' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='10.10.10.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='10.10.10.2' end='10.10.10.254' />
     </dhcp>
   </ip>
</network>

[root@fedora23wks ~]# cat external2.xml
<network>
   <name>external2</name>
   <uuid>d0a7964b-f93d-40c2-b749-b609aed52cf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr5' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='10.10.50.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='10.10.50.2' end='10.10.50.254' />
     </dhcp>
   </ip>
</network>

# virsh net-define external1.xml
# virsh net-start external1
# virsh net-autostart external1

# virsh net-define external2.xml
# virsh net-start external2
# virsh net-autostart external2

************************************************
Management network created via
************************************************
[root@fedora23wks ~]# cat openstackvms.xml
<network>
   <name>openstackvms</name>
   <uuid>d0e9964a-f91a-40c0-b769-a609aee41bf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr1' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='192.169.142.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='192.169.142.2' end='192.169.142.254' />
     </dhcp>
   </ip>
</network>
***********************************************
VTEPS network created via
***********************************************
[root@fedora23wks ~]# cat vteps.xml
<network>
   <name>vteps</name>
   <uuid>d2e9965b-f92c-40c1-b749-b609aed42cf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr2' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='12.0.0.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='12.0.0.1' end='12.0.0.254' />
     </dhcp>
   </ip>
</network>

*******************************************
Creating two flat networks as admin
*******************************************

# neutron net-create public1 --provider:network_type flat --provider:physical_network physnet1 --router:external --shared
# neutron net-create public2 --provider:network_type flat --provider:physical_network physnet2 --router:external --shared

# neutron subnet-create --gateway 10.10.10.1 --allocation-pool start=10.10.10.100,end=10.10.10.150 --disable-dhcp --name public1_subnet public1 10.10.10.0/24
# neutron subnet-create --gateway 10.10.50.1 --allocation-pool start=10.10.50.100,end=10.10.50.150 --disable-dhcp --name public2_subnet public2 10.10.50.0/24

***********************************************************************
On all nodes l3_agent.ini && openvswitch_agent.ini should
have highlighted entries
************************************************************************
[root@ip-192-169-142-127 neutron(keystone_admin)]# cat l3_agent.ini
[DEFAULT]
debug = False
interface_driver =neutron.agent.linux.interface.OVSInterfaceDriver
handle_internal_only_routers = True
external_network_bridge =
gateway_external_network_id =
metadata_port = 9697
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5
enable_metadata_proxy = True
router_delete_namespaces = False
agent_mode = dvr_snat
#on compute
# agent_mode =dvr
[AGENT]

[root@ip-192-169-142-127 ml2(keystone_admin)]# cat openvswitch_agent.ini
[ovs]
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip =12.0.0.127
network_vlan_ranges = physnet1,physnet2
bridge_mappings =physnet1:br-ex,physnet2:br-ex1
enable_tunneling=True
[agent]
polling_interval = 2
tunnel_types =vxlan
vxlan_udp_port =4789
l2_population = True
arp_responder = True
prevent_arp_spoofing = True
enable_distributed_routing = True
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

********************************************************************************
When updates above committed on all nodes && `openstack-service restart
neutron` also has been run , configure ifcfg-* files on all nodes as shown bellow.
Sequence of steps is important to allow service network successful restart.
********************************************************************************

[root@ip-192-169-142-137 network-scripts]# cat ifcfg-br-ex
DEVICE="br-ex"
NM_CONTROLLED="no"
ONBOOT="yes"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex
DEVICETYPE="ovs"

[root@ip-192-169-142-137 network-scripts]# cat ifcfg-br-ex1
DEVICE="br-ex1"
NM_CONTROLLED="no"
ONBOOT="yes"
TYPE="OVSIntPort"
OVS_BRIDGE=br-ex1
DEVICETYPE="ovs"

[root@ip-192-169-142-137 network-scripts]# cat ifcfg-eth2
DEVICE="eth2"
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex
NM_CONTROLLED=no
IPV6INIT=no

[root@ip-192-169-142-137 network-scripts]# cat ifcfg-eth3
DEVICE="eth3"
ONBOOT="yes"
TYPE="OVSPort"
DEVICETYPE="ovs"
OVS_BRIDGE=br-ex1
NM_CONTROLLED=no
IPV6INIT=no

*******************************************************************************
Now start two OVS bridges and two OVS ports ( eth2 belongs public1,
eth3 belongs public2 ) via script start.sh
*******************************************************************************
#!/bin/bash -x
chkconfig network on ;
systemctl stop NetworkManager ;
systemctl disable NetworkManager ;
service network restart

When done tune DVR configs per "RDO Liberty DVR Neutron workflow on CentOS 7.2" https://www.linux.com/community/blogs/133-general-linux/859376-rdo-liberty-rc2-dvr-neutron-workflow-on-centos-71
and restart nodes. Make sure VXLAN tunnels are present.

*****************
Verification
*****************

[root@ip-192-169-142-137 network-scripts]# ovs-vsctl show
cf8aca0d-1b18-4b88-8bdd-b5bdc5196176
    Bridge "br-ex1"
        Port "phy-br-ex1"                              <==== veth pair
            Interface "phy-br-ex1"
                type: patch
                options: {peer="int-br-ex1"}
        Port "eth3"
            Interface "eth3"
        Port "br-ex1"
            Interface "br-ex1"
                type: internal
    Bridge br-int
        fail_mode: secure
        Port "qr-a0a7d1ec-ad"
            tag: 3
            Interface "qr-a0a7d1ec-ad"
                type: internal
        Port "qvob03fdb03-fc"
            tag: 3
            Interface "qvob03fdb03-fc"
        Port int-br-ex
            Interface int-br-ex            <===== veth pair
                type: patch
                options: {peer=phy-br-ex}
        Port "fg-7b5266a1-03"      <==== outgoing interface of fip-namespace1
            tag: 4
            Interface "fg-7b5266a1-03"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "qvoc7a65c42-82"
            tag: 1
            Interface "qvoc7a65c42-82"
        Port "qr-034141ad-ae"
        tag: 1            Interface "qr-034141ad-ae"
                type: internal
        Port "fg-cfd8afb9-fe"      <==== outgoing interface of fip-namespace2
            tag: 2
            Interface "fg-cfd8afb9-fe"
                type: internal
        Port "int-br-ex1"
            Interface "int-br-ex1" <====    veth pair
                type: patch
                options: {peer="phy-br-ex1"}
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
    Bridge br-tun
        fail_mode: secure
        Port br-tun
            Interface br-tun
                type: internal
        Port "vxlan-0c00007f"
            Interface "vxlan-0c00007f"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="12.0.0.137", out_key=flow, remote_ip="12.0.0.127"}
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
        Port phy-br-ex
            Interface phy-br-ex     <==== veth pair
                type: patch
                options: {peer=int-br-ex}
        Port "eth2"
            Interface "eth2"
    ovs_version: "2.4.0"

***************************
Compute Node
***************************
[root@ip-192-169-142-137 ~]# ip netns
fip-393c5f85-e34a-4b0a-b5fc-582e464e9403
qrouter-7e33b300-104c-4415-bfbe-c212e4a45424
fip-15494dea-f35d-4572-a49c-9e8231cb9559
qrouter-83d7e1b2-125b-4d62-8cd7-eaf3cffe39c0

[root@ip-192-169-142-137 ~]# ip netns exec fip-393c5f85-e34a-4b0a-b5fc-582e464e9403 ip route
default via 10.10.10.1 dev fg-7b5266a1-03
10.10.10.0/24 dev fg-7b5266a1-03 proto kernel scope link src 10.10.10.102
10.10.10.101 via 169.254.31.238 dev fpr-7e33b300-1
169.254.31.238/31 dev fpr-7e33b300-1 proto kernel scope link src 169.254.31.239

[root@ip-192-169-142-137 ~]# ip netns exec fip-15494dea-f35d-4572-a49c-9e8231cb9559 ip route
default via 10.10.50.1 dev fg-cfd8afb9-fe
10.10.50.0/24 dev fg-cfd8afb9-fe proto kernel scope link src 10.10.50.102
10.10.50.101 via 169.254.31.28 dev fpr-83d7e1b2-1
169.254.31.28/31 dev fpr-83d7e1b2-1 proto kernel scope link src 169.254.31.29

[root@ip-192-169-142-137 ~]# ip netns exec qrouter-7e33b300-104c-4415-bfbe-c212e4a45424 ip route
70.0.0.0/24 dev qr-a0a7d1ec-ad proto kernel scope link src 70.0.0.1
169.254.31.238/31 dev rfp-7e33b300-1 proto kernel scope link src 169.254.31.238

[root@ip-192-169-142-137 ~]# ip netns exec qrouter-83d7e1b2-125b-4d62-8cd7-eaf3cffe39c0 ip route
50.0.0.0/24 dev qr-034141ad-ae proto kernel scope link src 50.0.0.1
169.254.31.28/31 dev rfp-83d7e1b2-1 proto kernel scope link src 169.254.31.28

[root@ip-192-169-142-137 ~]# ip netns exec fip-393c5f85-e34a-4b0a-b5fc-582e464e9403 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.239/31 scope global fpr-7e33b300-1
    inet6 fe80::6016:cfff:fed9:74c2/64 scope link
    inet 10.10.10.102/24 brd 10.10.10.255 scope global fg-7b5266a1-03
    inet6 fe80::f816:3eff:fe8d:431d/64 scope link

[root@ip-192-169-142-137 ~]# ip netns exec fip-15494dea-f35d-4572-a49c-9e8231cb9559 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.29/31 scope global fpr-83d7e1b2-1
    inet6 fe80::d8b1:1bff:fe28:264/64 scope link
    inet 10.10.50.102/24 brd 10.10.50.255 scope global fg-cfd8afb9-fe
    inet6 fe80::f816:3eff:feb5:cb7a/64 scope link

[root@ip-192-169-142-137 ~]# ip netns exec qrouter-7e33b300-104c-4415-bfbe-c212e4a45424 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.238/31 scope global rfp-7e33b300-1
    inet 10.10.10.101/32 brd 10.10.10.101 scope global rfp-7e33b300-1
    inet 10.10.10.103/32 brd 10.10.10.103 scope global rfp-7e33b300-1
    inet6 fe80::f6:d3ff:fecf:a5d5/64 scope link
    inet 70.0.0.1/24 brd 70.0.0.255 scope global qr-a0a7d1ec-ad
    inet6 fe80::f816:3eff:fe80:34a7/64 scope link

[root@ip-192-169-142-137 ~]# ip netns exec qrouter-83d7e1b2-125b-4d62-8cd7-eaf3cffe39c0 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.28/31 scope global rfp-83d7e1b2-1
    inet 10.10.50.103/32 brd 10.10.50.103 scope global rfp-83d7e1b2-1
    inet 10.10.50.101/32 brd 10.10.50.101 scope global rfp-83d7e1b2-1
    inet6 fe80::4c90:16ff:fe74:ec6d/64 scope link
    inet 50.0.0.1/24 brd 50.0.0.255 scope global qr-034141ad-ae
    inet6 fe80::f816:3eff:feae:1104/64 scope link

Three node deployment

   References
   1. http://blog.oddbit.com/2014/05/28/multiple-external-networks-wit/
   2. http://www.linux.com/community/blogs/133-general-linux/874900-running-dvr-with-external-network-provider-flat-on-centos-72-rdo-liberty/

Tuesday, December 22, 2015

Running DVR with External network provider (flat) on CentOS 7.2 RDO Liberty

Test bellow is targeting two potential problems :-
1. Creating HAProxy\Keepalived 3 Node Controller in RDO Mitaka with router
supporting VRRP && DVR at a time (coming up in Mitaka release) per https://github.com/beekhof/osp-ha-deploy/blob/master/HA-keepalived.md
in regards of using DVR on Compute nodes along with SNAT_DVR on HA 3 Node Controller.
   2. Creating DVR system working with two flat external networks. Details of conversion maybe seen in DVR with Two external networks via flat network provider on CentOS 7.2 RDO Liberty
Core tuning was done per http://blog.oddbit.com/2014/05/28/multiple-external-networks-wit/
Question which several times was raised up at ask.openstack.org, however
was not addressed properly.

*******************************************************************************
1. Setup Controller/Network + Compute ML2&OVS&VXLAN via answer-file
*******************************************************************************

[general]
CONFIG_SSH_KEY=/root/.ssh/id_rsa.pub
CONFIG_DEFAULT_PASSWORD=
CONFIG_MARIADB_INSTALL=y
CONFIG_GLANCE_INSTALL=y
CONFIG_CINDER_INSTALL=y
CONFIG_MANILA_INSTALL=n
CONFIG_NOVA_INSTALL=y
CONFIG_NEUTRON_INSTALL=y
CONFIG_HORIZON_INSTALL=y
CONFIG_SWIFT_INSTALL=y
CONFIG_CEILOMETER_INSTALL=y
CONFIG_SAHARA_INSTALL=n
CONFIG_HEAT_INSTALL=n
CONFIG_TROVE_INSTALL=n
CONFIG_IRONIC_INSTALL=n
CONFIG_CLIENT_INSTALL=y
CONFIG_NTP_SERVERS=
CONFIG_NAGIOS_INSTALL=n
EXCLUDE_SERVERS=
CONFIG_DEBUG_MODE=n
CONFIG_CONTROLLER_HOST=192.169.142.127
CONFIG_COMPUTE_HOSTS=192.169.142.137
CONFIG_NETWORK_HOSTS=192.169.142.127
CONFIG_VMWARE_BACKEND=n
CONFIG_UNSUPPORTED=n
CONFIG_USE_SUBNETS=n
CONFIG_VCENTER_HOST=
CONFIG_VCENTER_USER=
CONFIG_VCENTER_PASSWORD=
CONFIG_VCENTER_CLUSTER_NAME=
CONFIG_STORAGE_HOST=192.169.142.127
CONFIG_SAHARA_HOST=192.169.142.127
CONFIG_USE_EPEL=y
CONFIG_REPO=
CONFIG_ENABLE_RDO_TESTING=n
CONFIG_RH_USER=
CONFIG_SATELLITE_URL=
CONFIG_RH_PW=
CONFIG_RH_OPTIONAL=y
CONFIG_RH_PROXY=
CONFIG_RH_PROXY_PORT=
CONFIG_RH_PROXY_USER=
CONFIG_RH_PROXY_PW=
CONFIG_SATELLITE_USER=
CONFIG_SATELLITE_PW=
CONFIG_SATELLITE_AKEY=
CONFIG_SATELLITE_CACERT=
CONFIG_SATELLITE_PROFILE=
CONFIG_SATELLITE_FLAGS=
CONFIG_SATELLITE_PROXY=
CONFIG_SATELLITE_PROXY_USER=
CONFIG_SATELLITE_PROXY_PW=
CONFIG_SSL_CACERT_FILE=/etc/pki/tls/certs/selfcert.crt
CONFIG_SSL_CACERT_KEY_FILE=/etc/pki/tls/private/selfkey.key
CONFIG_SSL_CERT_DIR=~/packstackca/
CONFIG_SSL_CACERT_SELFSIGN=y
CONFIG_SELFSIGN_CACERT_SUBJECT_C=--
CONFIG_SELFSIGN_CACERT_SUBJECT_ST=State
CONFIG_SELFSIGN_CACERT_SUBJECT_L=City
CONFIG_SELFSIGN_CACERT_SUBJECT_O=openstack
CONFIG_SELFSIGN_CACERT_SUBJECT_OU=packstack
CONFIG_SELFSIGN_CACERT_SUBJECT_CN=ip-192-169-142-127.ip.secureserver.net
CONFIG_SELFSIGN_CACERT_SUBJECT_MAIL=admin@ip-192-169-142-127.ip.secureserver.net
CONFIG_AMQP_BACKEND=rabbitmq
CONFIG_AMQP_HOST=192.169.142.127
CONFIG_AMQP_ENABLE_SSL=n
CONFIG_AMQP_ENABLE_AUTH=n
CONFIG_AMQP_NSS_CERTDB_PW=PW_PLACEHOLDER
CONFIG_AMQP_AUTH_USER=amqp_user
CONFIG_AMQP_AUTH_PASSWORD=PW_PLACEHOLDER
CONFIG_MARIADB_HOST=192.169.142.127
CONFIG_MARIADB_USER=root
CONFIG_MARIADB_PW=7207ae344ed04957
CONFIG_KEYSTONE_DB_PW=abcae16b785245c3
CONFIG_KEYSTONE_DB_PURGE_ENABLE=True
CONFIG_KEYSTONE_REGION=RegionOne
CONFIG_KEYSTONE_ADMIN_TOKEN=3ad2de159f9649afb0c342ba57e637d9
CONFIG_KEYSTONE_ADMIN_EMAIL=root@localhost
CONFIG_KEYSTONE_ADMIN_USERNAME=admin
CONFIG_KEYSTONE_ADMIN_PW=7049f834927e4468
CONFIG_KEYSTONE_DEMO_PW=bf737b785cfa4398
CONFIG_KEYSTONE_API_VERSION=v2.0
CONFIG_KEYSTONE_TOKEN_FORMAT=UUID
CONFIG_KEYSTONE_SERVICE_NAME=httpd
CONFIG_KEYSTONE_IDENTITY_BACKEND=sql
CONFIG_KEYSTONE_LDAP_URL=ldap://192.169.142.127
CONFIG_KEYSTONE_LDAP_USER_DN=
CONFIG_KEYSTONE_LDAP_USER_PASSWORD=
CONFIG_KEYSTONE_LDAP_SUFFIX=
CONFIG_KEYSTONE_LDAP_QUERY_SCOPE=one
CONFIG_KEYSTONE_LDAP_PAGE_SIZE=-1
CONFIG_KEYSTONE_LDAP_USER_SUBTREE=
CONFIG_KEYSTONE_LDAP_USER_FILTER=
CONFIG_KEYSTONE_LDAP_USER_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_USER_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_MAIL_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_MASK=-1
CONFIG_KEYSTONE_LDAP_USER_ENABLED_DEFAULT=TRUE
CONFIG_KEYSTONE_LDAP_USER_ENABLED_INVERT=n
CONFIG_KEYSTONE_LDAP_USER_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_USER_DEFAULT_PROJECT_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_USER_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_USER_PASS_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_USER_ENABLED_EMULATION_DN=
CONFIG_KEYSTONE_LDAP_USER_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_GROUP_SUBTREE=
CONFIG_KEYSTONE_LDAP_GROUP_FILTER=
CONFIG_KEYSTONE_LDAP_GROUP_OBJECTCLASS=
CONFIG_KEYSTONE_LDAP_GROUP_ID_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_NAME_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_MEMBER_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_DESC_ATTRIBUTE=
CONFIG_KEYSTONE_LDAP_GROUP_ATTRIBUTE_IGNORE=
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_CREATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_UPDATE=n
CONFIG_KEYSTONE_LDAP_GROUP_ALLOW_DELETE=n
CONFIG_KEYSTONE_LDAP_GROUP_ADDITIONAL_ATTRIBUTE_MAPPING=
CONFIG_KEYSTONE_LDAP_USE_TLS=n
CONFIG_KEYSTONE_LDAP_TLS_CACERTDIR=
CONFIG_KEYSTONE_LDAP_TLS_CACERTFILE=
CONFIG_KEYSTONE_LDAP_TLS_REQ_CERT=demand
CONFIG_GLANCE_DB_PW=41264fc52ffd4fe8
CONFIG_GLANCE_KS_PW=f6a9398960534797
CONFIG_GLANCE_BACKEND=file
CONFIG_CINDER_DB_PW=5ac08c6d09ba4b69
CONFIG_CINDER_DB_PURGE_ENABLE=True
CONFIG_CINDER_KS_PW=c8cb1ecb8c2b4f6f
CONFIG_CINDER_BACKEND=lvm
CONFIG_CINDER_VOLUMES_CREATE=y
CONFIG_CINDER_VOLUMES_SIZE=5G
CONFIG_CINDER_GLUSTER_MOUNTS=
CONFIG_CINDER_NFS_MOUNTS=
CONFIG_CINDER_NETAPP_LOGIN=
CONFIG_CINDER_NETAPP_PASSWORD=
CONFIG_CINDER_NETAPP_HOSTNAME=
CONFIG_CINDER_NETAPP_SERVER_PORT=80
CONFIG_CINDER_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_CINDER_NETAPP_TRANSPORT_TYPE=http
CONFIG_CINDER_NETAPP_STORAGE_PROTOCOL=nfs
CONFIG_CINDER_NETAPP_SIZE_MULTIPLIER=1.0
CONFIG_CINDER_NETAPP_EXPIRY_THRES_MINUTES=720
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_START=20
CONFIG_CINDER_NETAPP_THRES_AVL_SIZE_PERC_STOP=60
CONFIG_CINDER_NETAPP_NFS_SHARES=
CONFIG_CINDER_NETAPP_NFS_SHARES_CONFIG=/etc/cinder/shares.conf
CONFIG_CINDER_NETAPP_VOLUME_LIST=
CONFIG_CINDER_NETAPP_VFILER=
CONFIG_CINDER_NETAPP_PARTNER_BACKEND_NAME=
CONFIG_CINDER_NETAPP_VSERVER=
CONFIG_CINDER_NETAPP_CONTROLLER_IPS=
CONFIG_CINDER_NETAPP_SA_PASSWORD=
CONFIG_CINDER_NETAPP_ESERIES_HOST_TYPE=linux_dm_mp
CONFIG_CINDER_NETAPP_WEBSERVICE_PATH=/devmgr/v2
CONFIG_CINDER_NETAPP_STORAGE_POOLS=
CONFIG_MANILA_DB_PW=PW_PLACEHOLDER
CONFIG_MANILA_KS_PW=PW_PLACEHOLDER
CONFIG_MANILA_BACKEND=generic
CONFIG_MANILA_NETAPP_DRV_HANDLES_SHARE_SERVERS=false
CONFIG_MANILA_NETAPP_TRANSPORT_TYPE=https
CONFIG_MANILA_NETAPP_LOGIN=admin
CONFIG_MANILA_NETAPP_PASSWORD=
CONFIG_MANILA_NETAPP_SERVER_HOSTNAME=
CONFIG_MANILA_NETAPP_STORAGE_FAMILY=ontap_cluster
CONFIG_MANILA_NETAPP_SERVER_PORT=443
CONFIG_MANILA_NETAPP_AGGREGATE_NAME_SEARCH_PATTERN=(.*)
CONFIG_MANILA_NETAPP_ROOT_VOLUME_AGGREGATE=
CONFIG_MANILA_NETAPP_ROOT_VOLUME_NAME=root
CONFIG_MANILA_NETAPP_VSERVER=
CONFIG_MANILA_GENERIC_DRV_HANDLES_SHARE_SERVERS=true
CONFIG_MANILA_GENERIC_VOLUME_NAME_TEMPLATE=manila-share-%s
CONFIG_MANILA_GENERIC_SHARE_MOUNT_PATH=/shares
CONFIG_MANILA_SERVICE_IMAGE_LOCATION=https://www.dropbox.com/s/vi5oeh10q1qkckh/ubuntu_1204_nfs_cifs.qcow2
CONFIG_MANILA_SERVICE_INSTANCE_USER=ubuntu
CONFIG_MANILA_SERVICE_INSTANCE_PASSWORD=ubuntu
CONFIG_MANILA_NETWORK_TYPE=neutron
CONFIG_MANILA_NETWORK_STANDALONE_GATEWAY=
CONFIG_MANILA_NETWORK_STANDALONE_NETMASK=
CONFIG_MANILA_NETWORK_STANDALONE_SEG_ID=
CONFIG_MANILA_NETWORK_STANDALONE_IP_RANGE=
CONFIG_MANILA_NETWORK_STANDALONE_IP_VERSION=4
CONFIG_MANILA_GLUSTERFS_SERVERS=
CONFIG_MANILA_GLUSTERFS_NATIVE_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_VOLUME_PATTERN=
CONFIG_MANILA_GLUSTERFS_TARGET=
CONFIG_MANILA_GLUSTERFS_MOUNT_POINT_BASE=
CONFIG_MANILA_GLUSTERFS_NFS_SERVER_TYPE=gluster
CONFIG_MANILA_GLUSTERFS_PATH_TO_PRIVATE_KEY=
CONFIG_MANILA_GLUSTERFS_GANESHA_SERVER_IP=
CONFIG_IRONIC_DB_PW=PW_PLACEHOLDER
CONFIG_IRONIC_KS_PW=PW_PLACEHOLDER
CONFIG_NOVA_DB_PURGE_ENABLE=True
CONFIG_NOVA_DB_PW=1e1b5aeeeaf342a8
CONFIG_NOVA_KS_PW=d9583177a2444f06
CONFIG_NOVA_SCHED_CPU_ALLOC_RATIO=16.0
CONFIG_NOVA_SCHED_RAM_ALLOC_RATIO=1.5
CONFIG_NOVA_COMPUTE_MIGRATE_PROTOCOL=tcp
CONFIG_NOVA_COMPUTE_MANAGER=nova.compute.manager.ComputeManager
CONFIG_VNC_SSL_CERT=
CONFIG_VNC_SSL_KEY=
CONFIG_NOVA_COMPUTE_PRIVIF=
CONFIG_NOVA_NETWORK_MANAGER=nova.network.manager.FlatDHCPManager
CONFIG_NOVA_NETWORK_PUBIF=eth0
CONFIG_NOVA_NETWORK_PRIVIF=
CONFIG_NOVA_NETWORK_FIXEDRANGE=192.168.32.0/22
CONFIG_NOVA_NETWORK_FLOATRANGE=10.3.4.0/22
CONFIG_NOVA_NETWORK_AUTOASSIGNFLOATINGIP=n
CONFIG_NOVA_NETWORK_VLAN_START=100
CONFIG_NOVA_NETWORK_NUMBER=1
CONFIG_NOVA_NETWORK_SIZE=255
CONFIG_NEUTRON_KS_PW=808e36e154bd4cee
CONFIG_NEUTRON_DB_PW=0e2b927a21b44737
CONFIG_NEUTRON_L3_EXT_BRIDGE=br-ex
CONFIG_NEUTRON_METADATA_PW=a965cd23ed2f4502
CONFIG_LBAAS_INSTALL=n
CONFIG_NEUTRON_METERING_AGENT_INSTALL=n
CONFIG_NEUTRON_FWAAS=n
CONFIG_NEUTRON_VPNAAS=n
CONFIG_NEUTRON_ML2_TYPE_DRIVERS=vxlan,flat
CONFIG_NEUTRON_ML2_TENANT_NETWORK_TYPES=vxlan
CONFIG_NEUTRON_ML2_MECHANISM_DRIVERS=openvswitch
CONFIG_NEUTRON_ML2_FLAT_NETWORKS=*
CONFIG_NEUTRON_ML2_VLAN_RANGES=
CONFIG_NEUTRON_ML2_TUNNEL_ID_RANGES=1001:2000
CONFIG_NEUTRON_ML2_VXLAN_GROUP=239.1.1.2
CONFIG_NEUTRON_ML2_VNI_RANGES=1001:2000
CONFIG_NEUTRON_L2_AGENT=openvswitch
CONFIG_NEUTRON_LB_INTERFACE_MAPPINGS=
CONFIG_NEUTRON_OVS_BRIDGE_MAPPINGS=physnet1:br-ex
CONFIG_NEUTRON_OVS_BRIDGE_IFACES=
CONFIG_NEUTRON_OVS_TUNNEL_IF=eth1
CONFIG_NEUTRON_OVS_VXLAN_UDP_PORT=4789
CONFIG_HORIZON_SSL=n
CONFIG_HORIZON_SECRET_KEY=a25b5ece9db24e2aba8d3a2b4d908ca5
CONFIG_HORIZON_SSL_CERT=
CONFIG_HORIZON_SSL_KEY=
CONFIG_HORIZON_SSL_CACERT=
CONFIG_SWIFT_KS_PW=8f75bfd461234c30
CONFIG_SWIFT_STORAGES=
CONFIG_SWIFT_STORAGE_ZONES=1
CONFIG_SWIFT_STORAGE_REPLICAS=1
CONFIG_SWIFT_STORAGE_FSTYPE=ext4
CONFIG_SWIFT_HASH=a60aacbedde7429a
CONFIG_SWIFT_STORAGE_SIZE=2G
CONFIG_HEAT_DB_PW=PW_PLACEHOLDER
CONFIG_HEAT_AUTH_ENC_KEY=976496a551b94296
CONFIG_HEAT_KS_PW=PW_PLACEHOLDER
CONFIG_HEAT_CLOUDWATCH_INSTALL=n
CONFIG_HEAT_CFN_INSTALL=n
CONFIG_HEAT_DOMAIN=heat
CONFIG_HEAT_DOMAIN_ADMIN=heat_admin
CONFIG_HEAT_DOMAIN_PASSWORD=PW_PLACEHOLDER
CONFIG_PROVISION_DEMO=y
CONFIG_PROVISION_TEMPEST=n
CONFIG_PROVISION_DEMO_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_IMAGE_NAME=cirros
CONFIG_PROVISION_IMAGE_URL=http://download.cirros-cloud.net/0.3.3/cirros-0.3.3-x86_64-disk.img
CONFIG_PROVISION_IMAGE_FORMAT=qcow2
CONFIG_PROVISION_IMAGE_SSH_USER=cirros
CONFIG_PROVISION_TEMPEST_USER=
CONFIG_PROVISION_TEMPEST_USER_PW=PW_PLACEHOLDER
CONFIG_PROVISION_TEMPEST_FLOATRANGE=172.24.4.224/28
CONFIG_PROVISION_TEMPEST_REPO_URI=https://github.com/openstack/tempest.git
CONFIG_PROVISION_TEMPEST_REPO_REVISION=master
CONFIG_PROVISION_OVS_BRIDGE=n
CONFIG_CEILOMETER_SECRET=19ae0e7430174349
CONFIG_CEILOMETER_KS_PW=337b08d4b3a44753
CONFIG_CEILOMETER_COORDINATION_BACKEND=redis
CONFIG_MONGODB_HOST=192.169.142.127
CONFIG_REDIS_MASTER_HOST=192.169.142.127
CONFIG_REDIS_PORT=6379
CONFIG_REDIS_HA=n
CONFIG_REDIS_SLAVE_HOSTS=
CONFIG_REDIS_SENTINEL_HOSTS=
CONFIG_REDIS_SENTINEL_CONTACT_HOST=
CONFIG_REDIS_SENTINEL_PORT=26379
CONFIG_REDIS_SENTINEL_QUORUM=2
CONFIG_REDIS_MASTER_NAME=mymaster
CONFIG_SAHARA_DB_PW=PW_PLACEHOLDER
CONFIG_SAHARA_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_DB_PW=PW_PLACEHOLDER
CONFIG_TROVE_KS_PW=PW_PLACEHOLDER
CONFIG_TROVE_NOVA_USER=trove
CONFIG_TROVE_NOVA_TENANT=services
CONFIG_TROVE_NOVA_PW=PW_PLACEHOLDER
CONFIG_NAGIOS_PW=PW_PLACEHOLDER

************************************************************************************
Three VNICs on each node MGMT (eth0) , VTEPS (eth1), EXT Interface (eth2)
************************************************************************************
Eth2 interfaces attached to VMs via libvirt subnet :-

[root@fedora23wks ~]# cat external1.xml
<network>
   <name>external1</name>
   <uuid>d0a7964b-f93d-40c2-b749-b609aed52cf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr4' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='10.10.10.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='10.10.10.2' end='10.10.10.254' />
     </dhcp>
   </ip>
</network>

# virsh net-define external1.xml
# virsh net-start external1
# virsh net-autostart external1

************************************************
Management network created via
************************************************
[root@fedora23wks ~]# cat openstackvms.xml
<network>
   <name>openstackvms</name>
   <uuid>d0e9964a-f91a-40c0-b769-a609aee41bf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr1' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='192.169.142.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='192.169.142.2' end='192.169.142.254' />
     </dhcp>
   </ip>
</network>
***********************************************
VTEPS network created via
***********************************************
[root@fedora23wks ~]# cat vteps.xml
<network>
   <name>vteps</name>
   <uuid>d2e9965b-f92c-40c1-b749-b609aed42cf2</uuid>
   <forward mode='nat'>
     <nat>
       <port start='1024' end='65535'/>
     </nat>
   </forward>
   <bridge name='virbr2' stp='on' delay='0' />
   <mac address='52:54:00:60:f8:6d'/>
   <ip address='12.0.0.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='12.0.0.1' end='12.0.0.254' />
     </dhcp>
   </ip>
</network>

*****************************************************************************
2. Tune all nodes to work with external network provider :-
*****************************************************************************
Set external_network_bridge = to an empty value in /etc/neutron/l3-agent.ini. This enables the use of external provider networks. Files ml2_conf.ini && openvswitch_agent.ini already tuned via answer-file directives. Then run

# openstack-service restart neutron

*****************************************************************************
On Controller/Network node create external flat network:-
******************************************************************************
[root@ip-192-169-142-127 ~(keystone_admin)]# neutron net-create public1 --provider:network_type flat --provider:physical_network physnet1 --router:external

[root@ip-192-169-142-127 ~(keystone_admin)]#    neutron subnet-create --gateway 10.10.10.1 --allocation-pool start=10.10.10.100,end=10.10.10.150 --disable-dhcp --name public1_subnet public1 10.10.10.0/24

******************
On all nodes
******************

# cat /etc/syscnfig/network-scripts/ifcfg-br-ex
    DEVICE="br-ex"
    NM_CONTROLLED="no"
    ONBOOT="yes"
    TYPE="OVSIntPort"
    OVS_BRIDGE=br-ex
    DEVICETYPE="ovs"

# cat /etc/syscnfig/network-scripts/ifcfg-eth2
    DEVICE="eth2"
    ONBOOT="yes"
    TYPE="OVSPort"
    DEVICETYPE="ovs"
    OVS_BRIDGE=br-ex
    NM_CONTROLLED=no
    IPV6INIT=no

# chkconfig network on
# systemctl stop NetworkManager
# systemctl disable NetworkManager
# service network restart

When done tune DVR configs per "RDO Liberty DVR Neutron workflow on CentOS 7.2" https://www.linux.com/community/blogs/133-general-linux/859376-rdo-liberty-rc2-dvr-neutron-workflow-on-centos-71
and restart nodes. Make sure VXLAN tunnels are present.

*********************************
Compute node configuration
*********************************

[root@ip-192-169-142-137 ~]# ip netns
fip-bb5509d1-84a3-489e-847f-c07573b8f6a1
qrouter-8a103913-f272-46ee-95de-38562860c3b1

[root@ip-192-169-142-137 ~]# ip netns exec fip-bb5509d1-84a3-489e-847f-c07573b8f6a1 ip route
default via 10.10.10.1 dev fg-a6949885-91
10.10.10.0/24 dev fg-a6949885-91 proto kernel scope link src 10.10.10.102
10.10.10.101 via 169.254.31.28 dev fpr-8a103913-f
10.10.10.103 via 169.254.31.28 dev fpr-8a103913-f
169.254.31.28/31 dev fpr-8a103913-f proto kernel scope link src 169.254.31.29

[root@ip-192-169-142-137 ~]# ip netns exec fip-bb5509d1-84a3-489e-847f-c07573b8f6a1 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.29/31 scope global fpr-8a103913-f
    inet6 fe80::44bd:31ff:fed2:b39f/64 scope link
    inet 10.10.10.102/24 brd 10.10.10.255 scope global fg-a6949885-91
    inet6 fe80::f816:3eff:fecf:84a5/64 scope link

[root@ip-192-169-142-137 ~]# ip netns exec qrouter-8a103913-f272-46ee-95de-38562860c3b1 ip a| grep "inet"
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host
    inet 169.254.31.28/31 scope global rfp-8a103913-f
    inet 10.10.10.101/32 brd 10.10.10.101 scope global rfp-8a103913-f
    inet 10.10.10.103/32 brd 10.10.10.103 scope global rfp-8a103913-f
    inet6 fe80::54be:36ff:fea5:918c/64 scope link
    inet 50.0.0.1/24 brd 50.0.0.255 scope global qr-98432f0d-0c
    inet6 fe80::f816:3eff:fe37:7da7/64 scope link

***************************************************************************************
Outgoing interface fg-a6949885-91 of fip-namespace is now attached to br-int.
Neutron flow is forwarded from fg-a6949885-91 to br-ex via veth pair
{phy-br-ex;int-br-ex} and gets outside through eth2 interface
***************************************************************************************

[root@ip-192-169-142-137 ~]# ovs-vsctl show
6b29bb4b-b7e0-42d7-94ba-662cd321bf82
    Bridge br-ex
        Port phy-br-ex
            Interface phy-br-ex                  <======= veth pair
                type: patch
                options: {peer=int-br-ex}
        Port br-ex
            Interface br-ex
                type: internal
        Port "eth2"
            Interface "eth2"
    Bridge br-int
        fail_mode: secure
        Port br-int
            Interface br-int
                type: internal
        Port "qvo997b88c5-a8"
            tag: 1
            Interface "qvo997b88c5-a8"
        Port int-br-ex
            Interface int-br-ex                  <========= veth pair
                type: patch
                options: {peer=phy-br-ex}
        Port "fg-a6949885-91"
            tag: 2
            Interface "fg-a6949885-91"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qvo2be937c0-cc"
            tag: 1
            Interface "qvo2be937c0-cc"
        Port "qr-98432f0d-0c"
            tag: 1
            Interface "qr-98432f0d-0c"
                type: internal
    Bridge br-tun
        fail_mode: secure
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
        Port "vxlan-0c00007f"
            Interface "vxlan-0c00007f"
                type: vxlan
                options: {df_default="true", in_key=flow, local_ip="12.0.0.137", out_key=flow, remote_ip="12.0.0.127"}
    ovs_version: "2.4.0"

**************
Controller
**************
[root@ip-192-169-142-127 ~(keystone_admin)]# ip netns exec qdhcp-e722c424-9f72-4236-a81f-77f79e097274 ip route
default via 50.0.0.1 dev tap7f80c809-9e
50.0.0.0/24 dev tap7f80c809-9e proto kernel scope link src 50.0.0.10

**************
Compute
**************
[root@ip-192-169-142-137 ~]# ip netns exec qrouter-8a103913-f272-46ee-95de-38562860c3b1 ip route
50.0.0.0/24 dev qr-98432f0d-0c proto kernel scope link src 50.0.0.1
169.254.31.28/31 dev rfp-8a103913-f proto kernel scope link src 169.254.31.28

[root@ip-192-169-142-137 ~]# ip netns exec fip-bb5509d1-84a3-489e-847f-c07573b8f6a1 ip route
default via 10.10.10.1 dev fg-a6949885-91
10.10.10.0/24 dev fg-a6949885-91 proto kernel scope link src 10.10.10.102
10.10.10.101 via 169.254.31.28 dev fpr-8a103913-f
10.10.10.103 via 169.254.31.28 dev fpr-8a103913-f
169.254.31.28/31 dev fpr-8a103913-f proto kernel scope link src 169.254.31.29

Compare with same report on Compute Nodes in
https://www.linux.com/community/blogs/133-general-linux/859376-rdo-liberty-rc2-dvr-neutron-workflow-on-centos-71
where fg-xxxxx interface is attached to bridge br-ex. Case of bridged external networking.

************************

On Compute node

************************

[root@ip-192-169-142-137 ~]# ip netns
fip-bb5509d1-84a3-489e-847f-c07573b8f6a1
qrouter-8a103913-f272-46ee-95de-38562860c3b1

Cloud VM VF23Devs01 is downloading 4.0 GB from Internet.
`iptop -i eth2` is running on Compute node console .

   *********************************
   On Controller/Network node
   *********************************
[root@ip-192-169-142-127 ~(keystone_admin)]# ip netns
    qdhcp-e722c424-9f72-4236-a81f-77f79e097274
    snat-8a103913-f272-46ee-95de-38562860c3b1
    qrouter-8a103913-f272-46ee-95de-38562860c3b1

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron router-port-list RouterDSA
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| id                                   | name | mac_address       | fixed_ips                                                                           |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| 98432f0d-0ce7-4ed0-84e4-427f3b70f359 |      | fa:16:3e:37:7d:a7 | {"subnet_id": "63727bd4-7586-4803-8cb1-c2a8b3cf990e", "ip_address": "50.0.0.1"}     |
| bc854d58-dd9c-4d88-9b9a-10fc69f2fbc4 |      | fa:16:3e:0d:99:24 | {"subnet_id": "a0935f2a-03ef-4ae9-902e-f791b95528fa", "ip_address": "10.10.10.100"} |
| caa27d49-8383-414f-ba29-39f73ac31ea0 |      | fa:16:3e:79:28:c4 | {"subnet_id": "63727bd4-7586-4803-8cb1-c2a8b3cf990e", "ip_address": "50.0.0.11"}    |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron l3-agent-list-hosting-router RouterDSA
+--------------------------------------+----------------------------------------+----------------+-------+----------+
| id                                   | host                                   | admin_state_up | alive | ha_state |
+--------------------------------------+----------------------------------------+----------------+-------+----------+
| 0d1cf08d-0d6c-4004-912f-eff90adc92a1 | ip-192-169-142-137.ip.secureserver.net | True           | :-)   |          |
| c42c97c0-e6a1-43a1-b1ed-f6e6c087b490 | ip-192-169-142-127.ip.secureserver.net | True           | :-)   |          |
+--------------------------------------+----------------------------------------+----------------+-------+----------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron net-list
+--------------------------------------+----------+----------------------------------------------------+
| id                                   | name     | subnets                                            |
+--------------------------------------+----------+----------------------------------------------------+
| bb5509d1-84a3-489e-847f-c07573b8f6a1 | public1 | a0935f2a-03ef-4ae9-902e-f791b95528fa 10.10.10.0/24 |
| e722c424-9f72-4236-a81f-77f79e097274 | demo_net | 63727bd4-7586-4803-8cb1-c2a8b3cf990e 50.0.0.0/24   |
+--------------------------------------+----------+----------------------------------------------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron net-show public1
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | bb5509d1-84a3-489e-847f-c07573b8f6a1 |
| mtu                       | 0                                    |
| name                      | public1                              |
| provider:network_type     | flat                                 |
| provider:physical_network | physnet1                             |
| provider:segmentation_id |                                      |
| router:external           | True                                 |
| shared                    | False                                |
| status                    | ACTIVE                               |
| subnets                   | a0935f2a-03ef-4ae9-902e-f791b95528fa |
| tenant_id                 | 2acd2c3b654f49e9a497dc1ad2807c9a     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron subnet-list
+--------------------------------------+----------------+---------------+--------------------------------------------------+
| id                                   | name           | cidr          | allocation_pools                                 |
+--------------------------------------+----------------+---------------+--------------------------------------------------+
| a0935f2a-03ef-4ae9-902e-f791b95528fa | public1_subnet | 10.10.10.0/24 | {"start": "10.10.10.100", "end": "10.10.10.150"} |
| 63727bd4-7586-4803-8cb1-c2a8b3cf990e | sub_demo_net   | 50.0.0.0/24   | {"start": "50.0.0.10", "end": "50.0.0.254"}      |
+--------------------------------------+----------------+---------------+--------------------------------------------------+

[root@ip-192-169-142-127 ~(keystone_admin)]# neutron subnet-show public1_subnet
+-------------------+--------------------------------------------------+
| Field             | Value                                            |
+-------------------+--------------------------------------------------+
| allocation_pools | {"start": "10.10.10.100", "end": "10.10.10.150"} |
| cidr              | 10.10.10.0/24                                    |
| dns_nameservers   |                                                  |
| enable_dhcp       | False                                            |
| gateway_ip        | 10.10.10.1                                       |
| host_routes       |                                                  |
| id                | a0935f2a-03ef-4ae9-902e-f791b95528fa             |
| ip_version        | 4                                                |
| ipv6_address_mode |                                                  |
| ipv6_ra_mode      |                                                  |
| name              | public1_subnet                                   |
| network_id        | bb5509d1-84a3-489e-847f-c07573b8f6a1             |
| subnetpool_id     |                                                  |
| tenant_id         | 2acd2c3b654f49e9a497dc1ad2807c9a                 |
+-------------------+--------------------------------------------------+

SNAT download via Controller/Network Node. Cloud VM VF23Devs02 is downloading 1.4 GB from Internet. `iptop -i eth2` is running on Controller
node console .

************************************************************
Final configuration on Controller/Network node
************************************************************
[root@ip-192-169-142-127 neutron(keystone_admin)]# cat l3_agent.ini | grep -v ^#|grep -v ^$
[DEFAULT]
debug = False
interface_driver =neutron.agent.linux.interface.OVSInterfaceDriver
handle_internal_only_routers = True
external_network_bridge =
gateway_external_network_id =
metadata_port = 9697
send_arp_for_ha = 3
periodic_interval = 40
periodic_fuzzy_delay = 5
enable_metadata_proxy = True
router_delete_namespaces = False
agent_mode = dvr_snat
[AGENT]

[root@ip-192-169-142-127 ml2(keystone_admin)]# cat ml2_conf.ini | grep -v ^#|grep -v ^$
[ml2]
type_drivers = vxlan,flat
tenant_network_types = vxlan
mechanism_drivers =openvswitch,l2population
path_mtu = 0
[ml2_type_flat]
flat_networks =*
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =1001:2000
vxlan_group =239.1.1.2
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

[root@ip-192-169-142-127 ml2(keystone_admin)]# cat openvswitch_agent.ini | grep -v ^#|grep -v ^$
[ovs]
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip =12.0.0.127
bridge_mappings =physnet1:br-ex
enable_tunneling=True
[agent]
polling_interval = 2
tunnel_types =vxlan
vxlan_udp_port =4789
l2_population = True
arp_responder = True
prevent_arp_spoofing = True
enable_distributed_routing = True
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

******************************************
Final configuration on Compute node
******************************************

[root@ip-192-169-142-137 neutron]# cat l3_agent.ini | grep -v ^#|grep -v ^$
[DEFAULT]
interface_driver = neutron.agent.linux.interface.OVSInterfaceDriver
external_network_bridge =
gateway_external_network_id =
agent_mode = dvr
[AGENT]

[root@ip-192-169-142-137 ml2]# cat ml2_conf.ini | grep -v ^#|grep -v ^$
[ml2]
type_drivers = vxlan,flat
tenant_network_types = vxlan
mechanism_drivers =openvswitch,l2population
path_mtu = 0
[ml2_type_flat]
flat_networks =*
[ml2_type_vlan]
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =1001:2000
vxlan_group =239.1.1.2
[ml2_type_geneve]
[securitygroup]
enable_security_group = True
[agent]
l2_population=True

[root@ip-192-169-142-137 ml2]# cat openvswitch_agent.ini | grep -v ^#|grep -v ^$
[ovs]
integration_bridge = br-int
tunnel_bridge = br-tun
local_ip =12.0.0.137
bridge_mappings =physnet1:br-ex
enable_tunneling=True
[agent]
polling_interval = 2
tunnel_types =vxlan
vxlan_udp_port =4789
l2_population = True
arp_responder = True
prevent_arp_spoofing = True
enable_distributed_routing = True
drop_flows_on_start=False
[securitygroup]
firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver

Sunday, December 13, 2015

AIO RDO Liberty && several external networks VLAN provider setup

UPDATE 05/04/2016
I got back to this writing due to
https://ask.openstack.org/en/question/91611/how-to-configure-multiple-external-networks-in-rdo-libertymitaka/
in answer field contains several misleading steps in configuration vlan enabled
OVS bridges.
END UPDATE

Post bellow is addressing the question when AIO RDO Liberty Node has to have external networks of VLAN type with predefined vlan tags. Straight forward packstack --allinone install doesn't allow to achieve desired network configuration. External network provider of vlan type appears to be required. In particular case, office networks 10.10.10.0/24 vlan tagged (157) ,10.10.57.0/24 vlan tagged (172), 10.10.32.0/24 vlan tagged (200) already exists when RDO install is running. If demo_provision was "y" , then delete router1 and created external network of VXLAN type

First
***********************************************************
Update /etc/neutron/plugins/ml2/ml2_conf.ini
***********************************************************
[root@ip-192-169-142-52 ml2(keystone_demo)]# cat ml2_conf.ini
[ml2]
type_drivers = flat,vlan,vxlan
tenant_network_types = vlan,vxlan
mechanism_drivers =openvswitch
path_mtu = 0
[ml2_type_flat]
[ml2_type_vlan]
network_vlan_ranges = vlan157:157:157,vlan172:172:172,vlan200:200:200
[ml2_type_gre]
[ml2_type_vxlan]
vni_ranges =10:100
vxlan_group =224.0.0.1
[ml2_type_geneve]
[securitygroup]
enable_security_group = True

**************
Then
**************

# openstack-service restart neutron

***************************************************
Invoke external network provider
***************************************************

[root@ip-192-169-142-52 ~(keystone_admin]#neutron net-create vlan157 --shared --provider:network_type vlan --provider:segmentation_id 157 --provider:physical_network vlan157 --router:external

[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan157 --gateway 10.10.10.1 --allocation-pool start=10.10.10.100,end=10.10.10.200 vlan157 10.10.10.0/24

***********************************************
Create second external network
***********************************************

[root@ip-192-169-142-52 ~(keystone_admin]# neutron net-create vlan172 --shared --provider:network_type vlan --provider:segmentation_id 172 --provider:physical_network vlan172  --router:external

[root@ip-192-169-142-52 ~(keystone_admin]# neutron subnet-create --name sub-vlan172 --gateway 10.10.57.1 --allocation-pool start=10.10.57.100,end=10.10.57.200 vlan172 10.10.57.0/24

***********************************************
Create third external network
***********************************************

[root@ip-192-169-142-52
 ~(keystone_admin]# neutron net-create vlan200 --shared 
--provider:network_type vlan --provider:segmentation_id 200 
--provider:physical_network vlan200  --router:external

[root@ip-192-169-142-52
 ~(keystone_admin]# neutron subnet-create --name sub-vlan200 --gateway 
10.10.32.1 --allocation-pool start=10.10.32.100,end=10.10.57.200 vlan172
 10.10.32.0/24

***********************************************************************
No need to update sub-net (vs [ 1 ]). No switch to "enable_isolataed_metadata=True"
Neutron L3 agent configuration results attaching qg-<port-id> interfaces to br-int
***********************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan157
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | b41e4d36-9a63-4631-abb0-6436f2f50e2e |
| mtu                       | 0                                    |
| name                      | vlan157                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan157                              |
| provider:segmentation_id | 157                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | bb753fc3-f257-4ce5-aa7c-56648648056b |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan157
+-------------------+------------------------------------------------------------------+
| Field             | Value                                                            |
+-------------------+------------------------------------------------------------------+
| allocation_pools | {"start": "10.10.10.100", "end": "10.10.10.200"}                 |
| cidr              | 10.10.10.0/24                                                    |
| dns_nameservers   |                                                                  |
| enable_dhcp       | True                                                             |
| gateway_ip        | 10.10.10.1                                                       |
| host_routes       | {"destination": "169.254.169.254/32", "nexthop": "10.10.10.151"} |
| id                | bb753fc3-f257-4ce5-aa7c-56648648056b                             |
| ip_version        | 4                                                                |
| ipv6_address_mode |                                                                  |
| ipv6_ra_mode      |                                                                  |
| name              | sub-vlan157                                                      |
| network_id        | b41e4d36-9a63-4631-abb0-6436f2f50e2e                             |
| subnetpool_id     |                                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                                 |
+-------------------+------------------------------------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan172
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3714adc9-ab17-4f96-9df2-48a6c0b64513 |
| mtu                       | 0                                    |
| name                      | vlan172                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan172                              |
| provider:segmentation_id | 172                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 21419f2f-212b-409a-8021-2b4a2ba6532f |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan172
+-------------------+------------------------------------------------------------------+
| Field             | Value                                                            |
+-------------------+------------------------------------------------------------------+
| allocation_pools | {"start": "10.10.57.100", "end": "10.10.57.200"}                 |
| cidr              | 10.10.57.0/24                                                    |
| dns_nameservers   |                                                                  |
| enable_dhcp       | True                                                             |
| gateway_ip        | 10.10.57.1                                                       |
| host_routes       | {"destination": "169.254.169.254/32", "nexthop": "10.10.57.151"} |
| id                | 21419f2f-212b-409a-8021-2b4a2ba6532f                             |
| ip_version        | 4                                                                |
| ipv6_address_mode |                                                                  |
| ipv6_ra_mode      |                                                                  |
| name              | sub-vlan172                                                      |
| network_id        | 3714adc9-ab17-4f96-9df2-48a6c0b64513                             |
| subnetpool_id     |                                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                                 |
+-------------------+------------------------------------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu                       | 0                                    |
| name                      | vlan200                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan200                              |
| provider:segmentation_id | 200                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-show sub-vlan200
+-------------------+--------------------------------------------------+
| Field             | Value                                            |
+-------------------+--------------------------------------------------+
| allocation_pools | {"start": "10.10.32.100", "end": "10.10.32.200"} |
| cidr              | 10.10.32.0/24                                    |
| dns_nameservers   |                                                  |
| enable_dhcp       | True                                             |
| gateway_ip        | 10.10.32.1                                       |
| host_routes       |                                                  |
| id                | 60181211-ea36-4e4e-8781-f13f743baa19             |
| ip_version        | 4                                                |
| ipv6_address_mode |                                                  |
| ipv6_ra_mode      |                                                  |
| name              | sub-vlan200                                      |
| network_id        | 3dc90ff7-b1df-4079-aca1-cceedb23f440             |
| subnetpool_id     |                                                  |
| tenant_id         | b18d25d66bbc48b1ad4b855a9c14da70                 |
+-------------------+--------------------------------------------------+

**************
Next Step
**************

# modprobe 8021q
# ovs-vsctl add-br br-vlan
# ovs-vsctl add-port br-vlan eth1
# vconfig add br-vlan 157

# ovs-vsctl add-br br-vlan2
# ovs-vsctl add-port br-vlan2 eth2
# vconfig add br-vlan2 172

# ovs-vsctl add-br br-vlan3
# ovs-vsctl add-port br-vlan3 eth3
# vconfig add br-vlan3 200

******************************
Update l3_agent.ini file
******************************
external_network_bridge =
gateway_external_network_id =

**********************************************
/etc/neutron/plugins/ml2/openvswitch_agent.ini
**********************************************
bridge_mappings = vlan157:br-vlan,vlan172:br-vlan2,vlan200:br-vlan3

*************************************
Update Neutron Configuration
*************************************

# openstack-service restart neutron

*******************************************
Set up config persistent between reboots
*******************************************
/etc/sysconfig/network-scripts/ifcfg-eth1

DEVICE="eth1"
ONBOOT=yes
OVS_BRIDGE=br-vlan
TYPE=OVSPort
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan

DEVICE=br-vlan
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan.157

BOOTPROTO="none"
DEVICE="br-vlan.157"
ONBOOT="yes"
IPADDR="10.10.10.150"
PREFIX="24"
GATEWAY="10.10.10.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes
USERCTL=no

/etc/sysconfig/network-scripts/ifcfg-eth2

DEVICE="eth2"
ONBOOT=yes
OVS_BRIDGE=br-vlan2
TYPE=OVSPort
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan2

DEVICE=br-vlan2
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan2.172

BOOTPROTO="none"
DEVICE="br-vlan2.172"
ONBOOT="yes"
IPADDR="10.10.57.150"
PREFIX="24"
GATEWAY="10.10.57.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes

/etc/sysconfig/network-scripts/ifcfg-br-vlan3
DEVICE=br-vlan3
BOOTPROTO=none
ONBOOT=yes
TYPE=OVSBridge
DEVICETYPE="ovs"

/etc/sysconfig/network-scripts/ifcfg-br-vlan3.200

BOOTPROTO="none"
DEVICE="br-vlan3.200"
ONBOOT="yes"
IPADDR="10.10.32.150"
PREFIX="24"
GATEWAY="10.10.32.1"
DNS1="83.221.202.254"
VLAN=yes
NOZEROCONF=yes
USERCTL=no

/etc/sysconfig/network-scripts/ifcfg-eth3

DEVICE="eth3"
ONBOOT=yes
OVS_BRIDGE=br-vlan3
TYPE=OVSPort
DEVICETYPE="ovs"

********************************************
Routing table on AIO RDO Liberty Node
********************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip route
default via 10.10.10.1 dev br-vlan.157
10.10.10.0/24 dev br-vlan.157 proto kernel scope link src 10.10.10.150
10.10.32.0/24 dev br-vlan3.200 proto kernel scope link src 10.10.32.150
10.10.57.0/24 dev br-vlan2.172 proto kernel scope link src 10.10.57.150
169.254.0.0/16 dev eth0 scope link metric 1002
169.254.0.0/16 dev eth1 scope link metric 1003
169.254.0.0/16 dev eth2 scope link metric 1004
169.254.0.0/16 dev eth3 scope link metric 1005
169.254.0.0/16 dev br-vlan3 scope link metric 1008
169.254.0.0/16 dev br-vlan2 scope link metric 1009
169.254.0.0/16 dev br-vlan scope link metric 1011
192.169.142.0/24 dev eth0 proto kernel scope link src 192.169.142.52

****************************************************************************
Notice that both qrouter-namespaces are attached to br-int.
No switch to "enable_isolated_metadata=True" vs [ 1 ]

*****************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-list | grep vlan
| 3dc90ff7-b1df-4079-aca1-cceedb23f440 | vlan200   | 60181211-ea36-4e4e-8781-f13f743baa19 10.10.32.0/24 |
| 235c8173-d3f8-407e-ad6a-c1d3d423c763 | vlan172   | c7588239-4941-419b-8d27-ccd970acc4ce 10.10.57.0/24 |
| b41e4d36-9a63-4631-abb0-6436f2f50e2e | vlan157   | bb753fc3-f257-4ce5-aa7c-56648648056b 10.10.10.0/24 |

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show
40286423-e174-4714-9c82-32d026ef47ca
    Bridge br-vlan
        Port "eth1"
            Interface "eth1"
        Port br-vlan
            Interface br-vlan
                type: internal
        Port phy-br-vlan
            Interface phy-br-vlan
                type: patch
                options: {peer=int-br-vlan}
    Bridge br-ex
        Port br-ex
            Interface br-ex
                type: internal
    Bridge br-tun
        fail_mode: secure
        Port br-tun
            Interface br-tun
                type: internal
        Port patch-int
            Interface patch-int
                type: patch
                options: {peer=patch-tun}
    Bridge "br-vlan2"
        Port "phy-br-vlan2"
            Interface "phy-br-vlan2"
                type: patch
                options: {peer="int-br-vlan2"}
        Port "eth2"
            Interface "eth2"
        Port "br-vlan2"
            Interface "br-vlan2"
                type: internal
    Bridge "br-vlan3"
        Port "br-vlan3"
            Interface "br-vlan3"
                type: internal
        Port "phy-br-vlan3"
            Interface "phy-br-vlan3"
                type: patch
                options: {peer="int-br-vlan3"}
        Port "eth3"
            Interface "eth3"
    Bridge br-int
        fail_mode: secure
        Port "qr-4e77c7a3-b5"
            tag: 3
            Interface "qr-4e77c7a3-b5"
                type: internal
        Port "int-br-vlan3"
            Interface "int-br-vlan3"
                type: patch
                options: {peer="phy-br-vlan3"}
        Port "tap8e684c78-a3"
            tag: 2
            Interface "tap8e684c78-a3"
                type: internal
        Port patch-tun
            Interface patch-tun
                type: patch
                options: {peer=patch-int}
        Port "qvoe2761636-b5"
            tag: 4
            Interface "qvoe2761636-b5"
        Port "tap6cd6fadf-31"
            tag: 1
            Interface "tap6cd6fadf-31"
                type: internal
        Port "qg-02f7ff0d-6d"
            tag: 2
            Interface "qg-02f7ff0d-6d"
                type: internal
        Port "qg-943f7831-46"
            tag: 1
            Interface "qg-943f7831-46"
                type: internal
        Port "tap4ef27b41-be"
            tag: 5
            Interface "tap4ef27b41-be"
                type: internal
        Port "qr-f0fd3793-4e"
            tag: 8
            Interface "qr-f0fd3793-4e"
                type: internal
        Port "tapb1435e62-8b"
            tag: 7
            Interface "tapb1435e62-8b"
                type: internal
        Port "qvo1bb76476-05"
            tag: 3
            Interface "qvo1bb76476-05"
        Port "qvocf68fcd8-68"
            tag: 8
            Interface "qvocf68fcd8-68"
        Port "qvo8605f075-25"
            tag: 4
            Interface "qvo8605f075-25"
        Port "qg-08ccc224-1e"
            tag: 7
            Interface "qg-08ccc224-1e"
                type: internal
        Port "tapbb485628-0b"
            tag: 4
            Interface "tapbb485628-0b"
                type: internal
        Port "int-br-vlan2"
            Interface "int-br-vlan2"
                type: patch
                options: {peer="phy-br-vlan2"}
        Port "tapee030534-da"
            tag: 8
            Interface "tapee030534-da"
                type: internal
        Port "qr-4d679697-39"
            tag: 4
            Interface "qr-4d679697-39"
                type: internal
        Port br-int
            Interface br-int
                type: internal
        Port "tap9b38c69e-46"
            tag: 6
            Interface "tap9b38c69e-46"
                type: internal
        Port "tapc166022a-54"
            tag: 3
            Interface "tapc166022a-54"
                type: internal
        Port "qvo66d8f235-d4"
            tag: 8
            Interface "qvo66d8f235-d4"
        Port int-br-vlan
            Interface int-br-vlan
                type: patch
                options: {peer=phy-br-vlan}
    ovs_version: "2.4.0"

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-e826aa22-dee0-478d-8bd7-721336e3824a
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-eda69965-c6ee-42be-944f-2d61498e4bea
qdhcp-6768214b-b71c-4178-a0fc-774b2a5d59ef
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qdhcp-03812cc9-69c5-492a-9995-985bf6e1ff13
qdhcp-235c8173-d3f8-407e-ad6a-c1d3d423c763
qdhcp-d958a059-f7bd-4f9f-93a3-3499d20a1fe2
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28
qrouter-71237c84-59ca-45dc-a6ec-23eb94c4249d

********************************************************************************
Access to Nova Metadata Server provided via neutron-ns-metadata-proxy
running in corresponding qrouter namespaces (Neutron L3 Configuration)
********************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b netstat -antp
Active Internet connections (servers and established)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name

tcp        0      0 0.0.0.0:9697            0.0.0.0:*               LISTEN      12548/python2

[root@ip-192-169-142-52 ~(keystone_admin)]# ps aux | grep 12548
neutron 12548 0.0 0.4 281028 35992 ?        S    18:34   0:00 /usr/bin/python2 /bin/neutron-ns-metadata-proxy --pid_file=/var/lib/neutron/external/pids/a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.pid --metadata_proxy_socket=/var/lib/neutron/metadata_proxy --router_id=a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b --state_path=/var/lib/neutron --metadata_port=9697 --metadata_proxy_user=990 --metadata_proxy_group=988 --verbose --log-file=neutron-ns-metadata-proxy-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b.log --log-dir=/var/log/neutron
root     32665 0.0 0.0 112644   960 pts/8    S+   19:29   0:00 grep --color=auto 12548

******************************************************************************
OVS flow verification on br-vlan3,br-vlan2. On each external network vlan172,
vlan200 two VMs (on each one of vlan networks) are pinging each other
******************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3554.739s, table=0, n_packets=33, n_bytes=2074, idle_age=2137, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4204.459s, table=0, n_packets=2102, n_bytes=109304, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3557.643s, table=0, n_packets=33, n_bytes=2074, idle_age=2140, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4207.363s, table=0, n_packets=2103, n_bytes=109356, idle_age=2, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=3568.225s, table=0, n_packets=33, n_bytes=2074, idle_age=2151, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=4217.945s, table=0, n_packets=2109, n_bytes=109668, idle_age=0, priority=0 actions=NORMAL

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4140.528s, table=0, n_packets=11, n_bytes=642, idle_age=695, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4225.918s, table=0, n_packets=2113, n_bytes=109876, idle_age=1, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4143.600s, table=0, n_packets=11, n_bytes=642, idle_age=698, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4228.990s, table=0, n_packets=2115, n_bytes=109980, idle_age=0, priority=0 actions=NORMAL
[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan2 | grep NORMAL
cookie=0x0, duration=4145.912s, table=0, n_packets=11, n_bytes=642, idle_age=700, priority=4,in_port=2,dl_vlan=1 actions=mod_vlan_vid:172,NORMAL
cookie=0x0, duration=4231.302s, table=0, n_packets=2116, n_bytes=110032, idle_age=0, priority=0 actions=NORMAL

********************************************************************************
Next question how local vlan tag 7 gets generated
Run following commands :-
********************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron net-show vlan200
+---------------------------+--------------------------------------+
| Field                     | Value                                |
+---------------------------+--------------------------------------+
| admin_state_up            | True                                 |
| id                        | 3dc90ff7-b1df-4079-aca1-cceedb23f440 |
| mtu                       | 0                                    |
| name                      | vlan200                              |
| provider:network_type     | vlan                                 |
| provider:physical_network | vlan200                              |
| provider:segmentation_id | 200                                  |
| router:external           | True                                 |
| shared                    | True                                 |
| status                    | ACTIVE                               |
| subnets                   | 60181211-ea36-4e4e-8781-f13f743baa19 |
| tenant_id                 | b18d25d66bbc48b1ad4b855a9c14da70     |
+---------------------------+--------------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
        inet 127.0.0.1 netmask 255.0.0.0
        inet6 ::1 prefixlen 128 scopeid 0x10<host>
        loop txqueuelen 0 (Local Loopback)
        RX packets 0 bytes 0 (0.0 B)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 0 bytes 0 (0.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
        inet 10.10.32.100 netmask 255.255.255.0 broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fee3:19f2 prefixlen 64 scopeid 0x20<link>
        ether fa:16:3e:e3:19:f2 txqueuelen 0 (Ethernet)
        RX packets 27 bytes 1526 (1.4 KiB)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 8 bytes 648 (648.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.32.1      0.0.0.0         UG    0      0        0 tapb1435e62-8b
10.10.32.0      0.0.0.0         255.255.255.0   U     0      0        0 tapb1435e62-8b

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-vsctl show | grep b1435e62-8b
        Port "tapb1435e62-8b"
            Interface "tapb1435e62-8b"
**************************************************************************
Actually, directives mentioned in [ 1 ]
**************************************************************************
# neutron subnet-create --name vlan100 --gateway 192.168.0.1 --allocation-pool \
start=192.168.0.150,end=192.168.0.200 --enable-dhcp \
--dns-nameserver 192.168.0.1 vlan100 192.168.0.0/24
# neutron subnet-update --host-route destination=169.254.169.254/32,nexthop=192.168.0.151 vlan100

along with switch to "enable_isolated_metadata=True" are targeting launching VMs to external_fixed_ip pool in qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 without creating Neutron router, spiting tenants with vlan tag IDs. I might be missing somesing , but [ 1 ] configures system where each vlan(XXX) external network would belong the only one tenant supposed identified by tag (XXX).
Unless RBAC policies will be created to control who has access to the provider network.

That is not what I intend to do. Neutron work flow on br-int won't touch mentioned qdhcp-namespace at all. Any external vlan(XXX) network might be used by several tenants each one having it ownVXLAN subnet (identified in system by VXLAN ID) and it's own neutron router(XXX) to external network vlan(XXX). AIO RDO set up is just a sample, I am talking about Network Node in multi node RDO Liberty depoyment.
*********************************************
Fragment from `ovs-vsct show `
*********************************************
Port "tapb1435e62-8b"
            tag: 7
            Interface "tapb1435e62-8b"

*************************************************************************
Next appearance of vlan tag 7, as expected is qg-08ccc224-1e.
Outgoing interface of qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
namespace.
*************************************************************************
[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
        inet 127.0.0.1 netmask 255.0.0.0
        inet6 ::1 prefixlen 128 scopeid 0x10<host>
        loop txqueuelen 0 (Local Loopback)
        RX packets 0 bytes 0 (0.0 B)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 0 bytes 0 (0.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

qg-08ccc224-1e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
        inet 10.10.32.101 netmask 255.255.255.0 broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fed4:e7d prefixlen 64 scopeid 0x20<link>
        ether fa:16:3e:d4:0e:7d txqueuelen 0 (Ethernet)
        RX packets 0 bytes 0 (0.0 B)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 28 bytes 1704 (1.6 KiB)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

qr-f0fd3793-4e: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
        inet 30.0.0.1 netmask 255.255.255.0 broadcast 30.0.0.255
        inet6 fe80::f816:3eff:fea9:5422 prefixlen 64 scopeid 0x20<link>
        ether fa:16:3e:a9:54:22 txqueuelen 0 (Ethernet)
        RX packets 68948 bytes 7192868 (6.8 MiB)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 68859 bytes 7185051 (6.8 MiB)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         10.10.32.1      0.0.0.0         UG    0      0        0 qg-08ccc224-1e
10.10.32.0      0.0.0.0         255.255.255.0   U     0      0        0 qg-08ccc224-1e
30.0.0.0        0.0.0.0         255.255.255.0   U     0      0        0 qr-f0fd3793-4e

*******************************************************************************************************
Now verify Neutron router connecting qrouter-namespace, having interface with tag 7 and qdhcp namespace, been create to launch the instances.
RoutesDSA has been created with external gateway to vlan200 and internal interface to subnet private07 (30.0.0.0/24) having dhcp enabled and DNS server defined.
vlan157,vlan172 are configured as external networks for theirs coresponding routers as well.
*******************************************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-list | grep RoutesDSA
| a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b | RoutesDSA | {"network_id": "3dc90ff7-b1df-4079-aca1-cceedb23f440", "enable_snat": true, "external_fixed_ips": [{"subnet_id": "60181211-ea36-4e4e-8781-f13f743baa19", "ip_address": "10.10.32.101"}]} | False       | False |

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b
qrouter-a2f4c7e8-9b63-4ed3-8d9a-faa6158d253b

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns | grep 3dc90ff7-b1df-4079-aca1-cceedb23f440
qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-3dc90ff7-b1df-4079-aca1-cceedb23f440 ifconfig
lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
        inet 127.0.0.1 netmask 255.0.0.0
        inet6 ::1 prefixlen 128 scopeid 0x10<host>
        loop txqueuelen 0 (Local Loopback)
        RX packets 0 bytes 0 (0.0 B)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 0 bytes 0 (0.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

tapb1435e62-8b: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1500
        inet 10.10.32.100 netmask 255.255.255.0 broadcast 10.10.32.255
        inet6 fe80::f816:3eff:fee3:19f2 prefixlen 64 scopeid 0x20<link>
        ether fa:16:3e:e3:19:f2 txqueuelen 0 (Ethernet)
        RX packets 27 bytes 1526 (1.4 KiB)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 8 bytes 648 (648.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

**************************
Finally run:-
**************************
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron router-port-list RoutesDSA
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| id                                   | name | mac_address       | fixed_ips                                                                           |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
| 08ccc224-1e23-491a-8eec-c4db0ec00f02 |      | fa:16:3e:d4:0e:7d | {"subnet_id": "60181211-ea36-4e4e-8781-f13f743baa19", "ip_address": "10.10.32.101"} |
| f0fd3793-4e5a-467a-bd3c-e87bc9063d26 |      | fa:16:3e:a9:54:22 | {"subnet_id": "0c962484-3e48-4d86-a17f-16b0b1e5fc4d", "ip_address": "30.0.0.1"}     |
+--------------------------------------+------+-------------------+-------------------------------------------------------------------------------------+
[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 0c962484-3e48-4d86-a17f-16b0b1e5fc4d
| 0c962484-3e48-4d86-a17f-16b0b1e5fc4d |               | 30.0.0.0/24   | {"start": "30.0.0.2", "end": "30.0.0.254"}       |

[root@ip-192-169-142-52 ~(keystone_admin)]# neutron subnet-list | grep 60181211-ea36-4e4e-8781-f13f743baa19
| 60181211-ea36-4e4e-8781-f13f743baa19 | sub-vlan200   | 10.10.32.0/24 | {"start": "10.10.32.100", "end": "10.10.32.200"} |

************************************
OVS Flows at br-vlan3
************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL

cookie=0x0, duration=15793.182s, table=0, n_packets=33, n_bytes=2074, idle_age=14376, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=16442.902s, table=0, n_packets=8221, n_bytes=427492, idle_age=1, priority=0 actions=NORMAL

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-flows br-vlan3 | grep NORMAL
cookie=0x0, duration=15796.300s, table=0, n_packets=33, n_bytes=2074, idle_age=14379, priority=4,in_port=2,dl_vlan=7 actions=mod_vlan_vid:200,NORMAL
cookie=0x0, duration=16446.020s, table=0, n_packets=8223, n_bytes=427596, idle_age=0, priority=0 actions=NORMAL

************************************************************
OVS Flow for {phy-br-vlan3,in-br-vlan3} veth pair
************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-vlan3 | grep phy-br-vlan3
2(phy-br-vlan3): addr:da:e4:fb:ba:8b:1a

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl show br-int | grep int-br-vlan3
19(int-br-vlan3): addr:b2:a9:9e:89:07:1b

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6977, bytes=304270, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6979, bytes=304354, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-vlan3 2
OFPST_PORT reply (xid=0x2): 1 ports
port 2: rx pkts=6981, bytes=304438, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=55, bytes=7037, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=6991, bytes=304858, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=6994, bytes=304984, drop=0, errs=0, coll=0

[root@ip-192-169-142-52 ~(keystone_admin)]# ovs-ofctl dump-ports br-int 19
OFPST_PORT reply (xid=0x2): 1 ports
port 19: rx pkts=55, bytes=7037, drop=0, errs=0, frame=0, over=0, crc=0
           tx pkts=7450, bytes=324136, drop=0, errs=0, coll=0

****************************************************************
Another OVS flow on test br-int for vlan157
****************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh -i oskeyvls.pem cirros@10.10.10.101
$ ping -c 5 10.10.10.108
PING 10.10.10.108 (10.10.10.108): 56 data bytes
64 bytes from 10.10.10.108: seq=0 ttl=63 time=0.706 ms
64 bytes from 10.10.10.108: seq=1 ttl=63 time=0.772 ms
64 bytes from 10.10.10.108: seq=2 ttl=63 time=0.734 ms
64 bytes from 10.10.10.108: seq=3 ttl=63 time=0.740 ms
64 bytes from 10.10.10.108: seq=4 ttl=63 time=0.785 ms

--- 10.10.10.108 ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max = 0.706/0.747/0.785 ms

******************************************************************************
   Testing VM1<=>VM2 via floating IPs on external vlan net 10.10.10.0/24
*******************************************************************************

[root@ip-192-169-142-52 ~(keystone_admin)]# nova list --all
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+
| ID                                   | Name         | Tenant ID                        | Status | Task State | Power State | Networks                        |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+
| a3d5ecf6-0fdb-4aa3-815f-171871eccb77 | CirrOSDevs01 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.17, 10.10.10.101 |
| 1b65f5db-d7d5-4e92-9a7c-60e7866ff8e5 | CirrOSDevs02 | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.18, 10.10.10.110 |
| 46b7dad1-3a7d-4d94-8407-a654cca42750 | VF23Devs01   | f16de8f8497d4f92961018ed836dee19 | ACTIVE | -          | Running     | private=40.0.0.19, 10.10.10.111 |
+--------------------------------------+--------------+----------------------------------+--------+------------+-------------+---------------------------------+

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns
qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20
qdhcp-b41e4d36-9a63-4631-abb0-6436f2f50e2e
qrouter-c1900dab-447f-4f87-80e7-b4c8ca087d28

[root@ip-192-169-142-52 ~(keystone_admin)]# ip netns exec qdhcp-4481aee1-ef86-4997-bf52-e435aafb9c20 ssh cirros@10.10.10.110
The authenticity of host '10.10.10.110 (10.10.10.110)' can't be established.
RSA key fingerprint is b8:d3:ec:10:70:a7:da:d4:50:13:a8:2d:01:ba:e4:83.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '10.10.10.110' (RSA) to the list of known hosts.
cirros@10.10.10.110's password:
$ ifconfig
eth0      Link encap:Ethernet HWaddr FA:16:3E:F1:6E:E5
          inet addr:40.0.0.18 Bcast:40.0.0.255 Mask:255.255.255.0
          inet6 addr: fe80::f816:3eff:fef1:6ee5/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST MTU:1400 Metric:1
          RX packets:367 errors:0 dropped:0 overruns:0 frame:0
          TX packets:291 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:36442 (35.5 KiB) TX bytes:32019 (31.2 KiB)

lo        Link encap:Local Loopback
          inet addr:127.0.0.1 Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING MTU:16436 Metric:1
          RX packets:0 errors:0 dropped:0 overruns:0 frame:0
          TX packets:0 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:0 (0.0 B) TX bytes:0 (0.0 B)

$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.110$

$ ssh fedora@10.10.10.111
Host '10.10.10.111' is not in the trusted hosts file.
(fingerprint md5 23:c0:fb:fd:74:80:2f:12:d3:09:2f:9e:dd:19:f1:74)
Do you want to continue connecting? (y/n) y
fedora@10.10.10.111's password:
Last login: Sun Dec 13 15:52:43 2015 from 10.10.10.101

[fedora@vf23devs01 ~]$ ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST> mtu 1400
        inet 40.0.0.19 netmask 255.255.255.0 broadcast 40.0.0.255
        inet6 fe80::f816:3eff:fea4:1a52 prefixlen 64 scopeid 0x20<link>
        ether fa:16:3e:a4:1a:52 txqueuelen 1000 (Ethernet)
        RX packets 283 bytes 30213 (29.5 KiB)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 303 bytes 35022 (34.2 KiB)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING> mtu 65536
        inet 127.0.0.1 netmask 255.0.0.0
        inet6 ::1 prefixlen 128 scopeid 0x10<host>
        loop txqueuelen 0 (Local Loopback)
        RX packets 0 bytes 0 (0.0 B)
        RX errors 0 dropped 0 overruns 0 frame 0
        TX packets 0 bytes 0 (0.0 B)
        TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/public-ipv4
10.10.10.111[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$ curl http://169.254.169.254/latest/meta-data/instance-id
i-00000009[fedora@vf23devs01 ~]$
[fedora@vf23devs01 ~]$