intergrate ns3 with mininet

 In my last blog[1], I intend to change link bandwidth dynamically in mininet. Even the method works, it still complains some error. So, I want to send the packets in ns3 for further processing, in which the delay and bandwidth of a link is easily manipulated. I follow the method in [2] and get it running.
 This code can be used to test multipath protocols (MPTCP, QUIC)
 Make some minor change in node.py.

--- a/mininet/node.py
+++ b/mininet/node.py
@@ -423,8 +423,11 @@ def addIntf( self, intf, port=None, moveIntfFn=moveIntf ):
         debug( 'added intf %s (%d) to node %s\n' % (
                 intf, port, self.name ) )
         if self.inNamespace:
-            debug( 'moving', intf, 'into namespace for', self.name, '\n' )
-            moveIntfFn( intf.name, self  )
+            if hasattr( intf, 'delayedMove' ) and intf.delayedMove is True:
+                pass
+            else:
+                debug( 'moving', intf, 'into namespace for', self.name, '\n' )
+                moveIntfFn( intf.name, self  )

     def defaultIntf( self ):
         "Return interface for lowest port"

 Then rebuild mininet.

mininet/util/install.sh

 Test code:
myns3.py

"""
NS-3 integration for Mininet.
 Mininet                                               Mininet
 node 1                                                node 2
+---------+                                           +---------+
| name    |                                           | name    |
| space 1 |                                           | space 2 |
| ------- |                                           |---------|
| shell   |                                           | shell   |
| ------- |                                           |---------|
| Linux   |                                           | Linux   |
| network |                                           | network |
| stack   |       ns-3              ns-3              | stack   |
| ------  |       node 1            node 2            |---------|
| TAP     |      |===========|     |===========|      | TAP     |
| intf.   |<-fd->| TapBridge |     | TapBridge |<-fd->| intf.   |
+---------+      | --------- |     | --------- |      +---------+
                 | ns-3      |     | ns-3      |
                 | net       |     | net       |
                 | device    |     | device    |
                 +-----------+     +-----------+
                       ||               ||
                  +---------------------------+
                  |        ns-3 channel       |
                  +---------------------------+
               |<------------------------------->|
                           ns-3 process
                      in the root namespace
"""

import threading, time

from mininet.log import info, error, warn, debug
from mininet.link import Intf, Link
from mininet.node import Switch, Node
from mininet.util import quietRun, moveIntf, errRun

import ns.core
import ns.network
import ns.tap_bridge
import ns.csma
import ns.wifi
import ns.mobility

# Default duration of ns-3 simulation thread. You can freely modify this value.

default_duration = 3600

# Set ns-3 simulator type to realtime simulator implementation.
# You can find more information about realtime modes here:
# http://www.nsnam.org/docs/release/3.17/manual/singlehtml/index.html#realtime
# http://www.nsnam.org/wiki/index.php/Emulation_and_Realtime_Scheduler

ns.core.GlobalValue.Bind( "SimulatorImplementationType", ns.core.StringValue( "ns3::RealtimeSimulatorImpl" ) )

# Enable checksum computation in ns-3 devices. By default ns-3 does not compute checksums - it is not needed
# when it runs in simulation mode. However, when it runs in emulation mode and exchanges packets with the real
# world, bit errors may occur in the real world, so we need to enable checksum computation.

ns.core.GlobalValue.Bind( "ChecksumEnabled", ns.core.BooleanValue ( "true" ) )

# Arrays which track all created TBIntf objects and Mininet nodes which has assigned an underlying ns-3 node.
#ns.core.LogComponentEnable("TapBridge", ns.core.LOG_LEVEL_INFO)
allTBIntfs = []
allNodes = []

# These four global functions below are used to control ns-3 simulator thread. They are global, because
# ns-3 has one global singleton simulator object.

def start():
    """ Start the simulator thread in background.
        It should be called after configuration of all ns-3 objects
        (TBintfs, Segments and Links).
        Attempt of adding an ns-3 object when simulator thread is
        running may result in segfault. You should stop it first."""
    global thread
    if 'thread' in globals() and thread.isAlive():
        warn( "NS-3 simulator thread already running." )
        return
    # Install all TapBridge ns-3 devices not installed yet.
    for intf in allTBIntfs:
        if not intf.nsInstalled:
            intf.nsInstall()
    # Set up the simulator thread.
    thread = threading.Thread( target = runthread )
    thread.daemon = True
    # Start the simulator thread (this is where fork happens).
    # FORK!
    thread.start()
    # FORK:PARENT
    # Code below is executed in the parent thread.
    # Move all tap interfaces not moved yet to the right namespace.
    for intf in allTBIntfs:
        if not intf.inRightNamespace:
            intf.namespaceMove()
    return

def runthread():
    """ Method called in the simulator thread on its start.
        Should not be called manually."""
    # FORK:CHILD
    # Code below is executed in the simulator thread after the fork.
    # Stop event must be scheduled before simulator start. Not scheduling it
    # may lead leads to segfault.
    ns.core.Simulator.Stop( ns.core.Seconds( default_duration ) )
    # Start simulator. Function below blocks the Python thread and returns when simulator stops.
    ns.core.Simulator.Run()

def stop():
    """ Stop the simulator thread now."""
    # Schedule a stop event.
    ns.core.Simulator.Stop( ns.core.MilliSeconds( 1 ) )
    # Wait until the simulator thread stops.
    while thread.isAlive():
        time.sleep( 0.01 )
    return

def clear():
    """ Clear ns-3 simulator.
        It should be called when simulator is stopped."""
    ns.core.Simulator.Destroy()
    for intf in allTBIntfs:
        intf.nsInstalled = False
        intf.delete()
    for node in allNodes:
        del node.nsNode
    del allTBIntfs[:]
    del allNodes[:]
    return

# Functions for manipulating nodes positions. Nodes positioning is useful in
# wireless channel simulations: distance between nodes affects received signal power
# and, thus, throughput.
# Node positions are stored in the underlying ns-3 node (not in Mininet node itself).

def getPosition( node ):
    """ Return the ns-3 (x, y, z) position of a Mininet node.
        Coordinates are in the 3D Cartesian system.
        The unit is meters.
        node: Mininet node"""
    # Check if this Mininet node has assigned the underlying ns-3 node.
    if hasattr( node, 'nsNode' ) and node.nsNode is not None:
        # If it is assigned, go ahead.
        pass
    else:
        # If not, create new ns-3 node and assign it to this Mininet node.
        node.nsNode = ns.network.Node()
        allNodes.append( node )
    try:
        # Get postion coordinates from the ns-3 node
        mm = node.nsNode.GetObject( ns.mobility.MobilityModel.GetTypeId() )
        pos = mm.GetPosition()
        return ( pos.x, pos.y, pos.z )
    except AttributeError:
        warn( "ns-3 mobility model not found\n" )
        return ( 0, 0, 0 )

def setPosition( node, x, y, z ):
    """ Set the ns-3 (x, y, z) position of a Mininet node.
        Coordinates are in the 3D Cartesian system.
        The unit is meters.
        node: Mininet node
        x: integer or float x coordinate
        y: integer or float y coordinate
        z: integer or float z coordinate"""
    # Check if this Mininet node has assigned the underlying ns-3 node.
    if hasattr( node, 'nsNode' ) and node.nsNode is not None:
        # If it is assigned, go ahead.
        pass
    else:
        # If not, create new ns-3 node and assign it to this Mininet node.
        node.nsNode = ns.network.Node()
        allNodes.append( node )
    try:
        mm = node.nsNode.GetObject( ns.mobility.MobilityModel.GetTypeId() )
        if z is None:
            z = 0.0
        # Set postion coordinates in the ns-3 node
        pos = mm.SetPosition( ns.core.Vector( x, y, z ) )
    except AttributeError:
        warn( "ns-3 mobility model not found, not setting position\n" )

# TBIntf is the main workhorse of the module. TBIntf is a tap Linux interface located on Mininet
# node, which is bridged with ns-3 device located on ns-3 node.
ip_addr="255.255.255."
mac_addr="93:85:45:6e:e5:"
ip_count=1
mac_count=1
class TBIntf( Intf ):
    """ Interface object that is bridged with ns-3 emulated device.
        This is a subclass of Mininet basic Inft object. """

    def __init__( self, name, node, port=None,
                  nsNode=None, nsDevice=None, mode=None, **params ):
        """name: interface name (e.g. h1-eth0)
           node: owning Mininet node (where this intf most likely lives)
           link: parent link if we're part of a link #TODO
           nsNode: underlying ns-3 node
           nsDevice: ns-3 device which the tap interface is bridged with
           mode: mode of TapBridge ns-3 device (UseLocal or UseBridge)
           other arguments are passed to config()"""
        self.name = name
        # Create a tap interface in the system, ns-3 TapBridge will connect to that interface later.
        self.createTap()
        # Set this Intf to be delayed move. This tells Mininet not to move the interface to the right
        # namespace during Intf.__init__(). Therefore, the interface must be moved manually later.
        # Actually, interfaces are moved right after the simulator thread start, in the start() global
        # function.
        self.delayedMove = True
        # If this node is running in its own namespace...
        if node.inNamespace:
            # ...this interface is not yet in the right namespace (it is in the root namespace just after
            # creation) and should be moved later.
            self.inRightNamespace = False
        else:
            # ...interface should stay in the root namespace, so it is in right namespace now.
            self.inRightNamespace = True
        # Initialize parent Intf object.
        Intf.__init__( self, name, node, port , **params)
        allTBIntfs.append( self )
        self.nsNode = nsNode
        self.nsDevice = nsDevice
        self.mode = mode
        self.params = params
        self.nsInstalled = False
        # Create TapBridge ns-3 device.
        self.tapbridge = ns.tap_bridge.TapBridge()
        # If ns-3 node and bridged ns-3 device are set and TapBridge mode is known...
        if self.nsNode and self.nsDevice and ( self.mode or self.node ):
            # ...call nsInstall().
            self.nsInstall()
    def eth_name(self):
        return self.name
    def createTap( self ):
        """Create tap Linux interface in the root namespace."""
        quietRun( 'ip tuntap add ' + self.name + ' mode tap' )

    def nsInstall( self ):
        """Install TapBridge ns-3 device in the ns-3 simulator."""
        if not isinstance( self.nsNode, ns.network.Node ):
            warn( "Cannot install TBIntf to ns-3 Node: "
                  "nsNode not specified\n" )
            return
        if not isinstance( self.nsDevice, ns.network.NetDevice ):
            warn( "Cannot install TBIntf to ns-3 Node: "
                  "nsDevice not specified\n" )
            return
        # If TapBridge mode has not been set explicitly, determine it automatically basing on
        # a Mininet node type. You can find more about TapBridge modes there:
        # http://www.nsnam.org/docs/release/3.18/models/singlehtml/index.html#tap-netdevice
        if self.mode is None and self.node is not None:
            # If Mininet node is some kind of Switch...
            if isinstance( self.node, Switch ):
                # ...use "UseBridge" mode. In this mode there may be many different L2 devices with
                # many source addresses on the Linux side of TapBridge, but bridged ns-3 device must
                # support SendFrom().
                self.mode = "UseBridge"
            else:
                # ...in the other case use "UseLocal" mode. In this mode there may be only one L2 source device
                # on the Linux side of TapBridge (TapBridge will change source MAC address of all packets coming
                # from the tap interface to the discovered address of this interface). In this mode bridged ns-3
                # device does not have to support SendFrom() (it uses Send() function to send packets).
                self.mode = "UseLocal"
        if self.mode is None:
            warn( "Cannot install TBIntf to ns-3 Node: "
                  "cannot determine mode: neither mode nor (mininet) node specified\n" )
            return
        # Set all required TapBridge attributes.
        #self.mode="ConfigureLocal"
        self.tapbridge.SetAttribute ( "Mode", ns.core.StringValue( self.mode ) )
        self.tapbridge.SetAttribute ( "DeviceName", ns.core.StringValue( self.name ) )
        #global ip_count
        #global mac_count
        #self.tapbridge.SetAttribute ( "IpAddress", ns.core.StringValue( ip_addr+str(ip_count)) )
        #self.tapbridge.SetAttribute ( "MacAddress", ns.core.StringValue( mac_addr+str(mac_count) ) )
        #ip_count+=1
        #mac_count+=1
        # Add TapBridge device to the ns-3 node.
        self.nsNode.AddDevice( self.tapbridge )
        # Set this TapBridge to be bridged with the specified ns-3 device.
        self.tapbridge.SetBridgedNetDevice( self.nsDevice )
        # Installation is done.
        self.nsInstalled = True

    def namespaceMove( self ):
        """Move tap Linux interface to the right namespace."""
        loops = 0
        # Wait until ns-3 process connects to the tap Linux interface. ns-3 process resides in the root
        # network namespace, so it must manage to connect to the interface before it is moved to the node
        # namespace. After interface move ns-3 process will not see the interface.
        while not self.isConnected():
            time.sleep( 0.01 )
            loops += 1
            if loops > 10:
                warn( "Cannot move TBIntf to mininet Node namespace: "
                      "ns-3 has not connected yet to the TAP interface\n" )
                return
        # Wait a little more, just for be sure ns-3 process not miss that.
        time.sleep( 0.1 )
        # Move interface to the right namespace.
        moveIntf( self.name, self.node )
        self.inRightNamespace = True
        # IP address has been reset while moving to namespace, needs to be set again.
        if self.ip is not None:
            self.setIP( self.ip, self.prefixLen )
        # The same for 'up'.
        self.isUp( True )

    def isConnected( self ):
        """Check if ns-3 TapBridge has connected to the Linux tap interface."""
        return self.tapbridge.IsLinkUp()

    def cmd( self, *args, **kwargs ):
        "Run a command in our owning node namespace or in the root namespace when not yet inRightNamespace."
        if self.inRightNamespace:
            return self.node.cmd( *args, **kwargs )
        else:
            cmd = ' '.join( [ str( c ) for c in args ] )
            return errRun( cmd )[ 0 ]

    def rename( self, newname ):
        "Rename interface"
        # If TapBridge is installed in ns-3, but ns-3 has not connected to the Linux tap interface yet...
        if self.nsInstalled and not self.isConnected():
            # ...change device name in TapBridge to the new one.
            self.tapbridge.SetAttribute ( "DeviceName", ns.core.StringValue( newname ) )
        Intf.rename( self, newname )

    def delete( self ):
        "Delete interface"
        if self.nsInstalled:
            warn( "You can not delete once installed ns-3 device, "
                  "run mininet.ns3.clear() to delete all ns-3 devices\n" )
        else:
            Intf.delete( self )

# Network segment is a Mininet object consistng of ns-3 channel of a specific type. This can be seen as
# an equivalent of collision domain. Many Mininet nodes can be connected to the one network segment.
# During connecting, Mininet creates ns-3 device of particular type in the underlying  ns-3 node.
# Then it connects this ns-3 device to the segment's ns-3 channel. Next, Mininet creates TBIntf in the
# specified Mininet node and bridges this tap interface with the ns-3 device created formerly.

# Network link is a subclass of network segment. It is a network segment with only two nodes connected.
# Moreover, network link is a subclass of Mininet Link. It means that you can use it like standard Mininet
# Link and alternatively with it: it supports all methods of its superclass and constructor arguments order
# is the same.

# SimpleChannel is the simplest channel model available in ns-3. Many devices can be connected to it
# simultaneously. Devices supports SendFrom(), therefore it can be used in "UseBridge" mode (for example
# for connecting switches). There is no implemented channel blocking - many devices can transmit
# simultaneously. Data rate and channel delay can not be set. However, one can
# set the receiver error model in SimpleNetDevice to simulate packet loss. You can find more information
# about the SimpleChannel in its source code and here:
# http://www.nsnam.org/docs/doxygen/classns3_1_1_simple_channel.html

class SimpleSegment( object ):
    """The simplest channel model available in ns-3.
       SimpleNetDevice supports SendFrom()."""
    def __init__( self,rate,delay):
        self.channel = ns.network.SimpleChannel()
        self.channel.SetAttribute("Delay", ns.core.TimeValue(ns.core.MilliSeconds (delay)))
        self.rate=rate
    def add( self, node, port=None, intfName=None, mode=None ):
        """Connect Mininet node to the segment.
           node: Mininet node
           port: node port number (optional)
           intfName: node tap interface name (optional)
           mode: TapBridge mode (UseLocal or UseBridge) (optional)"""
        # Check if this Mininet node has assigned an underlying ns-3 node.
        if hasattr( node, 'nsNode' ) and node.nsNode is not None:
            # If it is assigned, go ahead.
            pass
        else:
            # If not, create new ns-3 node and assign it to this Mininet node.
            node.nsNode = ns.network.Node()
            allNodes.append( node )
        # Create ns-3 device.
        device = ns.network.SimpleNetDevice()
        # Connect this device to the segment's channel.
        device.SetChannel(self.channel)
        device.SetAttribute("DataRate",ns.network.DataRateValue(ns.network.DataRate(self.rate)))
        # Add this device to the ns-3 node.
        node.nsNode.AddDevice(device)
        # If port number is not specified...
        if port is None:
            # ...obtain it automatically.
            port = node.newPort()
        # If interface name is not specified...
        if intfName is None:
            # ...obtain it automatically.
            ##intfName = Link.intfName( node, port ) # classmethod
            ##https://github.com/Barthurmun/NS3-Mininet/blob/master/mininet-patch/mininet/ns3.py
            intfName = node.name + '-eth' + repr( port )
        # In the specified Mininet node, create TBIntf bridged with the 'device'.
        tb = TBIntf( intfName, node, port, node.nsNode, device, mode )
        return tb


class SimpleLink( SimpleSegment, Link):
    """Link between two nodes using the SimpleChannel ns-3 model"""
    def __init__( self, node1, node2, rate,delay,port1=None, port2=None,
                  intfName1=None, intfName2=None ):
        """Create simple link to another node, making two new tap interfaces.
        node1: first Mininet node
        node2: second Mininet node
        port1: node1 port number (optional)
        port2: node2 port number (optional)
        intfName1: node1 interface name (optional)
        intfName2: node2 interface name (optional)"""
        SimpleSegment.__init__( self,rate,delay)
        intf1 = SimpleSegment.add( self, node1, port1, intfName1 )
        intf2 = SimpleSegment.add( self, node2, port2, intfName2 )
        intf1.link = self
        intf2.link = self
        self.intf1, self.intf2 = intf1, intf2
    def intf1_name(self):
        return self.intf1.eth_name()
    def intf2_name(self):
        return self.intf2.eth_name()
    def interface1(self):
        return self.intf1
    def interface2(self):
        return self.intf2       

simple-link.py

from mininet.net import Mininet
from mininet.node import Node, Switch
from mininet.link import Link, Intf
from mininet.log import setLogLevel, info
from mininet.cli import CLI
import time
import myns3
from myns3 import SimpleLink

##./waf shell
## python simple-link
##aborted. cond="bytesWritten != p->GetSize ()", msg="TapBridge::ReceiveFromBridgedDevice(): Write error.", 
##file=../src/tap-bridge/model/tap-bridge.cc, line=993 terminate called without an active exception
if __name__ == '__main__':
    setLogLevel( 'info' )
    info( '*** ns-3 network demo\n' )
    net = Mininet()

    info( '*** Creating Network\n' )
    h0 = Node( 'h0' )
    h1 = Node( 'h1' )

    net.hosts.append( h0 )
    net.hosts.append( h1 )
    bw=2000000
    link = SimpleLink( h0, h1,bw,10)   
    link1 = SimpleLink( h0, h1,bw,40 )
    myns3.start()

    info( '*** Configuring hosts\n' )
    print link.intf1_name(),link1.intf1_name()
    ip1='192.168.123.1'
    ip2='192.168.123.2'
    h0.setIP( ip1+'/24',intf=link.interface1())
    h1.setIP( ip2+'/24',intf=link.interface2())

    ip3='192.168.124.3'
    ip4='192.168.124.4'
    h0.setIP( ip3+'/24',intf=link1.interface1())
    h1.setIP( ip4+'/24',intf=link1.interface2())
    info( '*** Network state:\n' )
    for node in h0, h1:
        info( str( node ) + '\n' )

    info( '*** Running test\n' )
    h0.cmdPrint( 'ping -I'+ip1+' -c1 '+ ip2)
    h0.cmdPrint( 'ping -I'+ip3+' -c1 '+ ip4)
    time.sleep(1)
    h0.cmdPrint( 'ping -I'+ip1+' -c1 '+ ip2)
    h0.cmdPrint( 'ping -I'+ip3+' -c1 '+ ip4)
    h0.cmdPrint( 'ping -I'+ip1+' -c1 '+ ip2)
    h0.cmdPrint( 'ping -I'+ip3+' -c1 '+ ip4)
    #CLI(net)
    time.sleep(20)
    myns3.stop()

 Put simple-link.py and myns3.py under ns-allinone-3.2x/ns-3.2x.
 The topology in simple-link.py:

  h0-eth0-------h1-eth0
 /                    \
h0                    h1
 \                    /
  h0-eth1-------h1-eth1

  h0-eth0 and h0-eth1 should be configured with ip addresses in different subnets, or else there will be wrong results. Fuck it, this costs me one day!
 Run:

sudo su
./waf shell
./waf --pyrun  simple-link.py
mn -c

Reference
[1] change link bandwidth dynamically in mininet
[1] NS3-Mininet