#include <sys/types.h>
#include <sys/socket.h>

int socket(int domain, int type, int protocol);

PF_UNIX/PF_LOCAL/AF_UNIX/AF_LOCAL UNIX #进程通信协议
PF_INET?AF_INET Ipv4 #网络协议
PF_INET6/AF_INET6 Ipv6 #网络协议
PF_IPX/AF_IPX IPX-Novell #协议
PF_NETLINK/AF_NETLINK #核心用户接口装置
PF_X25/AF_X25 #ITU-T X.25/ISO-8208 协议
PF_AX25/AF_AX25 #业余无线AX. 25 协议
PF_ATMPVC/AF_ATMPVC #存取原始 ATM PVCs
PF_APPLETALK/AF_APPLETALK #appletalk (DDP)协议
PF_PACKET/AF_PACKET #初级封包接口

SOCK_STREAM #提供双向连续且可信赖的数据流, 即TCP. 支持 OOB 机制, 在所有数据传送前必须使用connect()来建立连线状态.
SOCK_DGRAM #使用不连续不可信赖的数据包连接
SOCK_SEQPACKET #提供连续可信赖的数据包连接
SOCK_RAW #提供原始网络协议存取
SOCK_RDM #提供可信赖的数据包连接
SOCK_PACKET #提供和网络驱动程序直接通信. protocol 用来指定socket 所使用的传输协议编号, 通常此参考不用管它, 设为0 即可.

#define ETH_P_ALL 0x0003
#define ETH_P_LOOP      0x0060          /* Ethernet Loopback packet     */      
#define ETH_P_PUP       0x0200          /* Xerox PUP packet             */      
#define ETH_P_PUPAT     0x0201          /* Xerox PUP Addr Trans packet  */      
#define ETH_P_IP        0x0800          /* Internet Protocol packet     */      
#define ETH_P_X25       0x0805          /* CCITT X.25                   */      
#define ETH_P_ARP       0x0806          /* Address Resolution packet    */      
#define ETH_P_BPQ       0x08FF          /* G8BPQ AX.25 Ethernet Packet  [ NOT AN
#define ETH_P_IEEEPUP   0x0a00          /* Xerox IEEE802.3 PUP packet */        
#define ETH_P_IEEEPUPAT 0x0a01          /* Xerox IEEE802.3 PUP Addr Trans packe[t*/

#define MAKE_RSPCODE(clas, det)((clas << 5) | det)

COAP_RSPCODE_CONTENT = MAKE_RSPCODE(2,5)

if (port_list) {
    add_ports(dp, port_list);
}

static void
add_ports(struct datapath *dp, char *port_list)
{
    char *port, *save_ptr; 
    /* Glibc 2.7 has a bug in strtok_r when compiling with optimization that
     * can cause segfaults here:
     * http://sources.redhat.com/bugzilla/show_bug.cgi?id=5614.
     * Using ",," instead of the obvious "," works around it. */
    for (port = strtok_r(port_list, ",,", &save_ptr); port;
         port = strtok_r(NULL, ",,", &save_ptr)) {
        int error = dp_add_port(dp, port, num_queues);
        if (error) {
            ofp_fatal(error, "failed to add port %s", port);
        }
    }
}

struct sw_port {
    uint32_t config;            /* Some subset of OFPPC_* flags. */
    uint32_t state;             /* Some subset of OFPPS_* flags. */
    struct datapath *dp;
    struct netdev *netdev;
    struct list node; /* Element in datapath.ports. */
    unsigned long long int rx_packets, tx_packets;
    unsigned long long int rx_bytes, tx_bytes;
    unsigned long long int tx_dropped;
    uint16_t port_no;
    /* port queues */
    uint16_t num_queues;
    struct sw_queue queues[NETDEV_MAX_QUEUES];
    struct list queue_list; /* list of all queues for this port */
};

int
dp_add_port(struct datapath *dp, const char *netdev, uint16_t num_queues)
{
    int port_no;
    for (port_no = 1; port_no < DP_MAX_PORTS; port_no++) {
        struct sw_port *port = &dp->ports[port_no];
        if (!port->netdev) {
            return new_port(dp, port, port_no, netdev, NULL, num_queues);
        }
    }
    return EXFULL;
}

static int
new_port(struct datapath *dp, struct sw_port *port, uint16_t port_no,
         const char *netdev_name, const uint8_t *new_mac, uint16_t num_queues)
{
    struct netdev *netdev;
    struct in6_addr in6;
    struct in_addr in4;
    int error;

    error = netdev_open(netdev_name, NETDEV_ETH_TYPE_ANY, &netdev);
    if (error) {
        return error;
    }
    if (new_mac && !eth_addr_equals(netdev_get_etheraddr(netdev), new_mac)) {
        /* Generally the device has to be down before we change its hardware
         * address.  Don't bother to check for an error because it's really
         * the netdev_set_etheraddr() call below that we care about. */
        netdev_set_flags(netdev, 0, false);
        error = netdev_set_etheraddr(netdev, new_mac);
        if (error) {
            VLOG_WARN("failed to change %s Ethernet address "
                      "to "ETH_ADDR_FMT": %s",
                      netdev_name, ETH_ADDR_ARGS(new_mac), strerror(error));
        }
    }
    error = netdev_set_flags(netdev, NETDEV_UP | NETDEV_PROMISC, false);
    if (error) {
        VLOG_ERR("failed to set promiscuous mode on %s device", netdev_name);
        netdev_close(netdev);
        return error;
    }
    if (netdev_get_in4(netdev, &in4)) {
        VLOG_ERR("%s device has assigned IP address %s",
                 netdev_name, inet_ntoa(in4));
    }
    if (netdev_get_in6(netdev, &in6)) {
        char in6_name[INET6_ADDRSTRLEN + 1];
        inet_ntop(AF_INET6, &in6, in6_name, sizeof in6_name);
        VLOG_ERR("%s device has assigned IPv6 address %s",
                 netdev_name, in6_name);
    }

    if (num_queues > 0) {
        error = netdev_setup_slicing(netdev, num_queues);
        if (error) {
            VLOG_ERR("failed to configure slicing on %s device: "\
                     "check INSTALL for dependencies, or rerun "\
                     "using --no-slicing option to disable slicing",
                     netdev_name);
            netdev_close(netdev);
            return error;
        }
    }

    memset(port, '\0', sizeof *port);

    list_init(&port->queue_list);
    port->dp = dp;
    port->netdev = netdev;
    port->port_no = port_no;
    port->num_queues = num_queues;
    list_push_back(&dp->port_list, &port->node);

    /* Notify the ctlpath that this port has been added */
    send_port_status(port, OFPPR_ADD);

    return 0;
}

/* Opens the network device named 'name' (e.g. "eth0") and returns zero if
 * successful, otherwise a positive errno value.  On success, sets '*netdevp'
 * to the new network device, otherwise to null.
 *
 * 'ethertype' may be a 16-bit Ethernet protocol value in host byte order to
 * capture frames of that type received on the device.  It may also be one of
 * the 'enum netdev_pseudo_ethertype' values to receive frames in one of those
 * categories. */
int
netdev_open(const char *name, int ethertype, struct netdev **netdevp)
{
    if (!strncmp(name, "tap:", 4)) {
        return netdev_open_tap(name + 4, netdevp);
    } else {
        return do_open_netdev(name, ethertype, -1, netdevp);
    }
}

enum netdev_pseudo_ethertype {
    NETDEV_ETH_TYPE_NONE = -128, /* Receive no frames. */
    NETDEV_ETH_TYPE_ANY,         /* Receive all frames. */
    NETDEV_ETH_TYPE_802_2        /* Receive all IEEE 802.2 frames. */
};

static int
do_open_netdev(const char *name, int ethertype, int tap_fd,
               struct netdev **netdev_)
{
    int netdev_fd;
    struct sockaddr_ll sll;
    struct ifreq ifr;
    unsigned int ifindex;
    uint8_t etheraddr[ETH_ADDR_LEN];
    struct in6_addr in6;
    int mtu;
    int txqlen;
    int hwaddr_family;
    int error;
    struct netdev *netdev;

    init_netdev();
    *netdev_ = NULL;

    /* Create raw socket. */
    netdev_fd = socket(PF_PACKET, SOCK_RAW,
                       htons(ethertype == NETDEV_ETH_TYPE_NONE ? 0
                             : ethertype == NETDEV_ETH_TYPE_ANY ? ETH_P_ALL
                             : ethertype == NETDEV_ETH_TYPE_802_2 ? ETH_P_802_2
                             : ethertype));
    if (netdev_fd < 0) {
        return errno;
    }

    /* Set non-blocking mode. */
    error = set_nonblocking(netdev_fd);
    if (error) {
        goto error_already_set;
    }

    /* Get ethernet device index. */
    strncpy(ifr.ifr_name, name, sizeof ifr.ifr_name);
    if (ioctl(netdev_fd, SIOCGIFINDEX, &ifr) < 0) {
        VLOG_ERR("ioctl(SIOCGIFINDEX) on %s device failed: %s",
                 name, strerror(errno));
        goto error;
    }
    ifindex = ifr.ifr_ifindex;

    /* Bind to specific ethernet device. */
    memset(&sll, 0, sizeof sll);
    sll.sll_family = AF_PACKET;
    sll.sll_ifindex = ifindex;
    if (bind(netdev_fd, (struct sockaddr *) &sll, sizeof sll) < 0) {
        VLOG_ERR("bind to %s failed: %s", name, strerror(errno));
        goto error;
    }

    if (ethertype != NETDEV_ETH_TYPE_NONE) {
        /* Between the socket() and bind() calls above, the socket receives all
         * packets of the requested type on all system interfaces.  We do not
         * want to receive that data, but there is no way to avoid it.  So we
         * must now drain out the receive queue. */
        error = drain_rcvbuf(netdev_fd);
        if (error) {
            goto error;
        }
    }

    /* Get MAC address. */
    if (ioctl(netdev_fd, SIOCGIFHWADDR, &ifr) < 0) {
        VLOG_ERR("ioctl(SIOCGIFHWADDR) on %s device failed: %s",
                 name, strerror(errno));
        goto error;
    }
    hwaddr_family = ifr.ifr_hwaddr.sa_family;
    if (hwaddr_family != AF_UNSPEC && hwaddr_family != ARPHRD_ETHER) {
        VLOG_WARN("%s device has unknown hardware address family %d",
                  name, hwaddr_family);
    }
    memcpy(etheraddr, ifr.ifr_hwaddr.sa_data, sizeof etheraddr);

    /* Get MTU. */
    if (ioctl(netdev_fd, SIOCGIFMTU, &ifr) < 0) {
        VLOG_ERR("ioctl(SIOCGIFMTU) on %s device failed: %s",
                 name, strerror(errno));
        goto error;
    }
    mtu = ifr.ifr_mtu;

    /* Get TX queue length. */
    if (ioctl(netdev_fd, SIOCGIFTXQLEN, &ifr) < 0) {
        VLOG_ERR("ioctl(SIOCGIFTXQLEN) on %s device failed: %s",
                 name, strerror(errno));
        goto error;
    }
    txqlen = ifr.ifr_qlen;

    get_ipv6_address(name, &in6);

    /* Allocate network device. */
    netdev = xmalloc(sizeof *netdev);
    netdev->name = xstrdup(name);
    netdev->ifindex = ifindex;
    netdev->txqlen = txqlen;
    netdev->hwaddr_family = hwaddr_family;
    netdev->netdev_fd = netdev_fd;
    netdev->tap_fd = tap_fd < 0 ? netdev_fd : tap_fd;
    netdev->queue_fd[0] = netdev->tap_fd;
    memcpy(netdev->etheraddr, etheraddr, sizeof etheraddr);
    netdev->mtu = mtu;
    netdev->in6 = in6;
    netdev->num_queues = 0;

    /* Get speed, features. */
    do_ethtool(netdev);

    /* Save flags to restore at close or exit. */
    error = get_flags(netdev->name, &netdev->save_flags);
    if (error) {
        goto error_already_set;
    }
    netdev->changed_flags = 0;
    fatal_signal_block();
    list_push_back(&netdev_list, &netdev->node);
    fatal_signal_unblock();

    /* Success! */
    *netdev_ = netdev;
    return 0;

error:
    error = errno;
error_already_set:
    close(netdev_fd);
    if (tap_fd >= 0) {
        close(tap_fd);
    }
    return error;
}

int
main(int argc, char *argv[])
{
    int n_listeners;
    int error;
    int i;
    set_program_name(argv[0]);  //设置程序名
    register_fault_handlers();  //注册新号故障处理器
    time_init();
    vlog_init();
    parse_options(argc, argv);  //解析选项
    signal(SIGPIPE, SIG_IGN);

  static void
parse_options(int argc, char *argv[])
{
...
    static struct option long_options[] = {
        {"interfaces",  required_argument, 0, 'i'},
        {"local-port",  required_argument, 0, 'L'},
        {"no-local-port", no_argument, 0, OPT_NO_LOCAL_PORT},
        {"datapath-id", required_argument, 0, 'd'},
        {"verbose",     optional_argument, 0, 'v'},
        {"help",        no_argument, 0, 'h'},
        {"version",     no_argument, 0, 'V'},
        {"no-slicing",  no_argument, 0, OPT_NO_SLICING},
        {"mfr-desc",    required_argument, 0, OPT_MFR_DESC},
        {"hw-desc",     required_argument, 0, OPT_HW_DESC},
        {"sw-desc",     required_argument, 0, OPT_SW_DESC},
        {"dp_desc",  required_argument, 0, OPT_DP_DESC},
        {"serial_num",  required_argument, 0, OPT_SERIAL_NUM},
        DAEMON_LONG_OPTIONS,

    for (;;) {
        int indexptr;
        int c;

        c = getopt_long(argc, argv, short_options, long_options, &indexptr);
        if (c == -1) {
            break;
        }

        switch (c) {
        case 'd':
            if (strlen(optarg) != 12
                || strspn(optarg, "0123456789abcdefABCDEF") != 12) {
                ofp_fatal(0, "argument to -d or --datapath-id must be "
                          "exactly 12 hex digits");
            }
            dpid = strtoll(optarg, NULL, 16);
            if (!dpid) {
                ofp_fatal(0, "argument to -d or --datapath-id must "
                          "be nonzero");
            }
            break;

        case 'h':
            usage();

        case 'V':
            printf("%s %s compiled "__DATE__" "__TIME__"\n",
                   program_name, VERSION BUILDNR);
            exit(EXIT_SUCCESS);

        case 'v':
            vlog_set_verbosity(optarg);
            break;

        case 'i':
            if (!port_list) {
                port_list = optarg;
            } else {
                port_list = xasprintf("%s,%s", port_list, optarg);
            }
            break;

        case 'L':
            local_port = optarg;
            break;

        case OPT_NO_LOCAL_PORT:
            local_port = NULL;
            break;

        case OPT_MFR_DESC:
            strncpy(mfr_desc, optarg, sizeof mfr_desc);
            break;

        case OPT_HW_DESC:
            strncpy(hw_desc, optarg, sizeof hw_desc);
            break;

        case OPT_SW_DESC:
            strncpy(sw_desc, optarg, sizeof sw_desc);
            break;

        case OPT_DP_DESC:
            strncpy(dp_desc, optarg, sizeof dp_desc);
            break;

        case OPT_SERIAL_NUM:
            strncpy(serial_num, optarg, sizeof serial_num);
            break;

        case OPT_NO_SLICING:
            num_queues = 0;
            break;

        DAEMON_OPTION_HANDLERS

        }
    }
    free(short_options);
}

if (argc - optind < 1) {
    ofp_fatal(0, "at least one listener argument is required; "
      "use --help for usage");
}

error = dp_new(&dp, dpid);

int
dp_new(struct datapath **dp_, uint64_t dpid)
{
    struct datapath *dp;

    dp = calloc(1, sizeof *dp);
    if (!dp) {
        return ENOMEM;
    }

    dp->last_timeout = time_now();
    list_init(&dp->remotes);
    dp->listeners = NULL;
    dp->n_listeners = 0;
    dp->id = dpid <= UINT64_C(0xffffffffffff) ? dpid : gen_datapath_id();
    dp->chain = chain_create(dp);  //Creates and returns a new chain.
    if (!dp->chain) {
        VLOG_ERR("could not create chain");
        free(dp);
        return ENOMEM;
    }

    list_init(&dp->port_list);
    dp->flags = 0;
    dp->miss_send_len = OFP_DEFAULT_MISS_SEND_LEN;

    if(strlen(&dp_desc) > 0)	/* use the comment, if specified */
	    strncpy(dp->dp_desc, &dp_desc, sizeof dp->dp_desc);
    else			            /* else, just use "$HOSTNAME pid=$$" */
    {
        char hostnametmp[DESC_STR_LEN];
	    gethostname(hostnametmp,sizeof hostnametmp);
        snprintf(dp->dp_desc, sizeof dp->dp_desc,"%s pid=%u",hostnametmp, getpid());
    }

    *dp_ = dp;
    return 0;
}

/* Creates and returns a new chain.  Returns NULL if the chain cannot be
 * created. */
struct sw_chain *chain_create(struct datapath *dp)
{
    struct sw_chain *chain = calloc(1, sizeof *chain); //分配内存空间
    if (chain == NULL)
        return NULL;

    chain->dp = dp;
    if (add_table(chain, table_hash2_create(0x1EDC6F41, TABLE_HASH_MAX_FLOWS,
                                            0x741B8CD7, TABLE_HASH_MAX_FLOWS),
                                            0)
        || add_table(chain, table_linear_create(TABLE_LINEAR_MAX_FLOWS), 0)
        || add_table(chain, table_linear_create(TABLE_LINEAR_MAX_FLOWS), 1)) {
        chain_destroy(chain);
        return NULL;
    }

    return chain;
}

/* Attempts to append 'table' to the set of tables in 'chain'.  Returns 0 or
 * negative error.  If 'table' is null it is assumed that table creation failed
 * due to out-of-memory. */
static int add_table(struct sw_chain *chain, struct sw_table *table, int emerg)
{
    if (table == NULL)
        return -ENOMEM;
    if (chain->n_tables >= CHAIN_MAX_TABLES) {
        VLOG_ERR("too many tables in chain\n");
        table->destroy(table);
        return -ENOBUFS;
    }
    if (emerg)
        chain->emerg_table = table;
    else
        chain->tables[chain->n_tables++] = table;
    return 0;
}

n_listeners = 0;
for (i = optind; i < argc; i++) {
    const char *pvconn_name = argv[i];
    struct pvconn *pvconn;
    int retval;

    retval = pvconn_open(pvconn_name, &pvconn);
    if (!retval || retval == EAGAIN) {
        dp_add_pvconn(dp, pvconn);
        n_listeners++;
    } else {
        ofp_error(retval, "opening %s", pvconn_name);
    }
}
if (!n_listeners) {
    ofp_fatal(0, "could not listen for any connections");
}

if (port_list) {
    add_ports(dp, port_list);
}
if (local_port) {
    error = dp_add_local_port(dp, local_port, 0);
    if (error) {
        ofp_fatal(error, "failed to add local port %s", local_port);
    }
}

error = vlog_server_listen(NULL, NULL);
if (error) {
    ofp_fatal(error, "could not listen for vlog connections");
}

die_if_already_running();
daemonize();

void
daemonize(void)
{
    if (detach) {  //detach是一个bool值，判断是否作为后台程序运行
        char c = 0;
        int fds[2];
        if (pipe(fds) < 0) {
            ofp_fatal(errno, "pipe failed");
        }

        switch (fork()) {
        default:
            /* Parent process: wait for child to create pidfile, then exit. */
            close(fds[1]);
            fatal_signal_fork();
            if (read(fds[0], &c, 1) != 1) {
                ofp_fatal(errno, "daemon child failed to signal startup");
            }
            exit(0);

        case 0:
            /* Child process. */
            close(fds[0]);
            make_pidfile();  //进程文件
            write(fds[1], &c, 1);  //读写操作
            close(fds[1]);
            setsid();
            chdir("/");
            break;

        case -1:
            /* Error. */
            ofp_fatal(errno, "could not fork");
            break;
        }
    } else {
        make_pidfile();
    }
}

/*add the monitor function*/
tp = realloc(tp, sizeof(struct thread_para));
tp->dp = dp;
make_socket(tp);

struct thread_para {
    struct border_port *bp;
    struct datapath *dp;
};

//server socket using UDP
int make_socket(struct thread_para *tp) {
    
    printf("enter make socket function\n");
    const int SERV_PORT = 5555;
    pthread_t tid;
    int sockfd;
    struct sockaddr_in servaddr;

    bzero(&servaddr , sizeof(servaddr));
    servaddr.sin_family = AF_INET;
    servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
    servaddr.sin_port = htons(SERV_PORT);

    if((sockfd = socket(AF_INET , SOCK_DGRAM , 0)) < 0)
    {
        perror("socket error");
        exit(1);
    }
    //_fd = sockfd;
    struct border_port* bp;
    bp = realloc(bp, sizeof(struct border_port));
    tp->bp = bp;
    tp->bp->fd = sockfd; 

    if(bind(sockfd , (struct sockaddr *)&servaddr , sizeof(servaddr)))
    {
        perror("bind error");
        exit(1);
    }

    printf("create server socket\n");
    //create thread to send and receive message from click module
    if( pthread_create(&tid, NULL, socket_handler, tp) != 0 )
        pdie("pthread_create");
    
    return 0;
}

for (;;) {
    dp_run(dp);
    dp_wait(dp);
    poll_block();
}

void
dp_run(struct datapath *dp)
{
    time_t now = time_now();
    struct sw_port *p, *pn;
    struct remote *r, *rn;
    struct ofpbuf *buffer = NULL;
    size_t i;

    if (now != dp->last_timeout) {
        struct list deleted = LIST_INITIALIZER(&deleted);
        struct sw_flow *f, *n;

        chain_timeout(dp->chain, &deleted);
        LIST_FOR_EACH_SAFE (f, n, struct sw_flow, node, &deleted) {
            dp_send_flow_end(dp, f, f->reason);
            list_remove(&f->node);
            flow_free(f);
        }
        dp->last_timeout = now;
    }
    poll_timer_wait(1000);

/*
 * 以上对超时做了排错处理
 * dp_send_flow_end 将错误代码发送到控制器，具体分析见下。
 */
    
    LIST_FOR_EACH_SAFE (p, pn, struct sw_port, node, &dp->port_list) {
        int error;

        if (!buffer) {
            /* Allocate buffer with some headroom to add headers in forwarding
             * to the controller or adding a vlan tag, plus an extra 2 bytes to
             * allow IP headers to be aligned on a 4-byte boundary.  */
            const int headroom = 128 + 2;
            const int hard_header = VLAN_ETH_HEADER_LEN;
            const int mtu = netdev_get_mtu(p->netdev);
            buffer = ofpbuf_new(headroom + hard_header + mtu);
            buffer->data = (char*)buffer->data + headroom;
        }
        
 /*
  * buffer用来暂时存储netdevice的数据包
  * ofbuf_new 定义了一个空的buffer空间
  */
        error = netdev_recv(p->netdev, buffer);
        if (!error) {
            p->rx_packets++;
            p->rx_bytes += buffer->size;
            fwd_port_input(dp, buffer, p);
            buffer = NULL;
        } else if (error != EAGAIN) {
            VLOG_ERR_RL(&rl, "error receiving data from %s: %s",
                        netdev_get_name(p->netdev), strerror(error));
        }
    }
    ofpbuf_delete(buffer);

    /*
     * 建立与Controller之间的安全信道
   	 */

    /* Talk to remotes. */
    LIST_FOR_EACH_SAFE (r, rn, struct remote, node, &dp->remotes) {
        remote_run(dp, r);
    }

    for (i = 0; i < dp->n_listeners; ) {
        struct pvconn *pvconn = dp->listeners[i];
        struct vconn *new_vconn;
        int retval = pvconn_accept(pvconn, OFP_VERSION, &new_vconn);
        if (!retval) {
            remote_create(dp, rconn_new_from_vconn("passive", new_vconn));
        } else if (retval != EAGAIN) {
            VLOG_WARN_RL(&rl, "accept failed (%s)", strerror(retval));
            dp->listeners[i] = dp->listeners[--dp->n_listeners];
            continue;
        }
        i++;
    }
}

struct ofpbuf *
ofpbuf_new(size_t size)
{
    struct ofpbuf *b = xmalloc(sizeof *b);
    ofpbuf_init(b, size);
    return b;
}

int
netdev_recv(struct netdev *netdev, struct ofpbuf *buffer)
{
    ssize_t n_bytes;
    struct sockaddr_ll sll;
    socklen_t sll_len;

    assert(buffer->size == 0);
    assert(ofpbuf_tailroom(buffer) >= ETH_TOTAL_MIN);

    /* prepare to call recvfrom */
    memset(&sll,0,sizeof sll);
    sll_len = sizeof sll;

    /* cannot execute recvfrom over a tap device */
    if (!strncmp(netdev->name, "tap", 3)) {
        do {
            n_bytes = read(netdev->tap_fd, ofpbuf_tail(buffer),
                           (ssize_t)ofpbuf_tailroom(buffer));
        } while (n_bytes < 0 && errno == EINTR);
    }
    else {
        do {
            n_bytes = recvfrom(netdev->tap_fd, ofpbuf_tail(buffer),
                               (ssize_t)ofpbuf_tailroom(buffer), 0,
                               (struct sockaddr *)&sll, &sll_len);
        } while (n_bytes < 0 && errno == EINTR);
    }
    if (n_bytes < 0) {
        if (errno != EAGAIN) {
            VLOG_WARN_RL(&rl, "error receiving Ethernet packet on %s: %s",
                         strerror(errno), netdev->name);
        }
        return errno;
    } else {
        /* we have multiple raw sockets at the same interface, so we also
         * receive what others send, and need to filter them out.
         * TODO(yiannisy): can we install this as a BPF at kernel? */
        if (sll.sll_pkttype == PACKET_OUTGOING) {
            return EAGAIN;
        }


        buffer->size += n_bytes;

        /* When the kernel internally sends out an Ethernet frame on an
         * interface, it gives us a copy *before* padding the frame to the
         * minimum length.  Thus, when it sends out something like an ARP
         * request, we see a too-short frame.  So pad it out to the minimum
         * length. */
        pad_to_minimum_length(buffer);
        return 0;
    }
}

/* 'buffer' was received on 'p', which may be a a physical switch port or a
 * null pointer.  Process it according to 'dp''s flow table, sending it up to
 * the controller if no flow matches.  Takes ownership of 'buffer'. */
void fwd_port_input(struct datapath *dp, struct ofpbuf *buffer,
                    struct sw_port *p)
{
    if (run_flow_through_tables(dp, buffer, p)) {
        dp_output_control(dp, buffer, p->port_no,
                          dp->miss_send_len, OFPR_NO_MATCH);
    }
}

/* 'buffer' was received on 'p', which may be a a physical switch port or a
 * null pointer.  Process it according to 'dp''s flow table.  Returns 0 if
 * successful, in which case 'buffer' is destroyed, or -ESRCH if there is no
 * matching flow, in which case 'buffer' still belongs to the caller. */
int run_flow_through_tables(struct datapath *dp, struct ofpbuf *buffer,
                            struct sw_port *p)
{
    struct sw_flow_key key;
    struct sw_flow *flow;

    key.wildcards = 0;
    if (flow_extract(buffer, p ? p->port_no : OFPP_NONE, &key.flow)
        && (dp->flags & OFPC_FRAG_MASK) == OFPC_FRAG_DROP) {
        /* Drop fragment. */
        ofpbuf_delete(buffer);
        return 0;
    }

    if (p && p->config & (OFPPC_NO_RECV | OFPPC_NO_RECV_STP)
        && p->config & (!eth_addr_equals(key.flow.dl_dst, stp_eth_addr)
                       ? OFPPC_NO_RECV : OFPPC_NO_RECV_STP)) {
        ofpbuf_delete(buffer);
        return 0;
    }

    flow = chain_lookup(dp->chain, &key, 0);
    if (flow != NULL) {
        flow_used(flow, buffer);
        execute_actions(dp, buffer, &key, flow->sf_acts->actions,
                        flow->sf_acts->actions_len, false);
        return 0;
    } else {
        return -ESRCH;
    }
}

struct sw_flow {
    struct sw_flow_key key;

    uint64_t cookie;            /* Opaque controller-issued identifier. */
    uint16_t priority;          /* Only used on entries with wildcards. */
    uint16_t idle_timeout;      /* Idle time before discarding (seconds). */
    uint16_t hard_timeout;      /* Hard expiration time (seconds) */
    uint64_t used;              /* Last used time. */
    uint64_t created;           /* When the flow was created. */
    uint64_t packet_count;      /* Number of packets seen. */
    uint64_t byte_count;        /* Number of bytes seen. */
    uint8_t reason;             /* Reason flow removed (one of OFPRR_*). */
    uint8_t send_flow_rem;      /* Send a flow removed to the controller */
    uint8_t emerg_flow;         /* Emergency flow indicator */

    struct sw_flow_actions *sf_acts;

    /* Private to table implementations. */
    struct list node;
    struct list iter_node;
    unsigned long int serial;
};

/* Returns 1 if 'packet' is an IP fragment, 0 otherwise. */
int
flow_extract(struct ofpbuf *packet, uint16_t in_port, struct flow *flow)
{
    struct ofpbuf b = *packet;
    struct eth_header *eth;
    int retval = 0;

    memset(flow, 0, sizeof *flow);
    flow->dl_vlan = htons(OFP_VLAN_NONE);
    flow->in_port = htons(in_port);

    packet->l2 = b.data;
    packet->l3 = NULL;
    packet->l4 = NULL;
    packet->l7 = NULL;

    eth = pull_eth(&b);
    if (eth) {
        if (ntohs(eth->eth_type) >= OFP_DL_TYPE_ETH2_CUTOFF) {
            /* This is an Ethernet II frame */
            flow->dl_type = eth->eth_type;
        } else {
            /* This is an 802.2 frame */
            struct llc_header *llc = ofpbuf_at(&b, 0, sizeof *llc);
            struct snap_header *snap = ofpbuf_at(&b, sizeof *llc,
                                                 sizeof *snap);
            if (llc == NULL) {
                return 0;
            }
            if (snap
                && llc->llc_dsap == LLC_DSAP_SNAP
                && llc->llc_ssap == LLC_SSAP_SNAP
                && llc->llc_cntl == LLC_CNTL_SNAP
                && !memcmp(snap->snap_org, SNAP_ORG_ETHERNET,
                           sizeof snap->snap_org)) {
                flow->dl_type = snap->snap_type;
                ofpbuf_pull(&b, LLC_SNAP_HEADER_LEN);
            } else {
                flow->dl_type = htons(OFP_DL_TYPE_NOT_ETH_TYPE);
                ofpbuf_pull(&b, sizeof(struct llc_header));
            }
        }

        /* Check for a VLAN tag */
        if (flow->dl_type == htons(ETH_TYPE_VLAN)) {
            struct vlan_header *vh = pull_vlan(&b);
            if (vh) {
                flow->dl_type = vh->vlan_next_type;
                flow->dl_vlan = vh->vlan_tci & htons(VLAN_VID_MASK);
                flow->dl_vlan_pcp = (uint8_t)((ntohs(vh->vlan_tci) >> VLAN_PCP_SHIFT)
                                               & VLAN_PCP_BITMASK);
            }
        }
        memcpy(flow->dl_src, eth->eth_src, ETH_ADDR_LEN);
        memcpy(flow->dl_dst, eth->eth_dst, ETH_ADDR_LEN);

        packet->l3 = b.data;
        if (flow->dl_type == htons(ETH_TYPE_IP)) {
            const struct ip_header *nh = pull_ip(&b);
            if (nh) {
                flow->nw_tos = nh->ip_tos & 0xfc;
                flow->nw_proto = nh->ip_proto;
                flow->nw_src = nh->ip_src;
                flow->nw_dst = nh->ip_dst;
                packet->l4 = b.data;
                if (!IP_IS_FRAGMENT(nh->ip_frag_off)) {
                    if (flow->nw_proto == IP_TYPE_TCP) {
                        const struct tcp_header *tcp = pull_tcp(&b);
                        if (tcp) {
                            flow->tp_src = tcp->tcp_src;
                            flow->tp_dst = tcp->tcp_dst;
                            packet->l7 = b.data;
                        } else {
                            /* Avoid tricking other code into thinking that
                             * this packet has an L4 header. */
                            flow->nw_proto = 0;
                        }
                    } else if (flow->nw_proto == IP_TYPE_UDP) {
                        const struct udp_header *udp = pull_udp(&b);
                        if (udp) {
                            flow->tp_src = udp->udp_src;
                            flow->tp_dst = udp->udp_dst;
                            packet->l7 = b.data;
                        } else {
                            /* Avoid tricking other code into thinking that
                             * this packet has an L4 header. */
                            flow->nw_proto = 0;
                        }
                    } else if (flow->nw_proto == IP_TYPE_ICMP) {
                        const struct icmp_header *icmp = pull_icmp(&b);
                        if (icmp) {
                            flow->icmp_type = htons(icmp->icmp_type);
                            flow->icmp_code = htons(icmp->icmp_code);
                            packet->l7 = b.data;
                        } else {
                            /* Avoid tricking other code into thinking that
                             * this packet has an L4 header. */
                            flow->nw_proto = 0;
                        }
                    }
                } else {
                    retval = 1;
                }
            }
        } else if (flow->dl_type == htons(ETH_TYPE_ARP)) {
            const struct arp_eth_header *arp = pull_arp(&b);
            if (arp) {
                if (arp->ar_pro == htons(ARP_PRO_IP) && arp->ar_pln == IP_ADDR_LEN) {
                    flow->nw_src = arp->ar_spa;
                    flow->nw_dst = arp->ar_tpa;
                }
                flow->nw_proto = ntohs(arp->ar_op) && 0xff;
            }
        }
    }
    return retval;
}

/* Searches 'chain' for a flow matching 'key', which must not have any wildcard
 * fields.  Returns the flow if successful, otherwise a null pointer. */
struct sw_flow *
chain_lookup(struct sw_chain *chain, const struct sw_flow_key *key, int emerg)
{
    int i;

    assert(!key->wildcards);

    if (emerg) {
        struct sw_table *t = chain->emerg_table;
        struct sw_flow *flow = t->lookup(t, key);
        t->n_lookup++;
        if (flow) {
            t->n_matched++;
            return flow;
        }
    } else {
        for (i = 0; i < chain->n_tables; i++) {
            struct sw_table *t = chain->tables[i];
            struct sw_flow *flow = t->lookup(t, key);
            t->n_lookup++;
            if (flow) {
                t->n_matched++;
                return flow;
            }
        }
    }

    return NULL;
}

/* Execute a list of actions against 'buffer'. */
void execute_actions(struct datapath *dp, struct ofpbuf *buffer,
             struct sw_flow_key *key,
             const struct ofp_action_header *actions, size_t actions_len,
             int ignore_no_fwd)
{
    /* Every output action needs a separate clone of 'buffer', but the common
     * case is just a single output action, so that doing a clone and then
     * freeing the original buffer is wasteful.  So the following code is
     * slightly obscure just to avoid that. */
    int prev_port;
    uint32_t prev_queue;
    size_t max_len = UINT16_MAX;
    uint16_t in_port = ntohs(key->flow.in_port);
    uint8_t *p = (uint8_t *)actions;

    prev_port = -1;
    prev_queue = 0;

    /* The action list was already validated, so we can be a bit looser
     * in our sanity-checking. */
    while (actions_len > 0) {
        struct ofp_action_header *ah = (struct ofp_action_header *)p;
        size_t len = htons(ah->len);

        if (prev_port != -1) {
            do_output(dp, ofpbuf_clone(buffer), in_port, max_len,
                      prev_port, prev_queue, ignore_no_fwd);
            prev_port = -1;
        }

        if (ah->type == htons(OFPAT_OUTPUT)) {
            struct ofp_action_output *oa = (struct ofp_action_output *)p;
            prev_port = ntohs(oa->port);
            prev_queue = 0; /* using the default best-effort queue */
            max_len = ntohs(oa->max_len);
        } else if (ah->type == htons(OFPAT_ENQUEUE)) {
            struct ofp_action_enqueue *ea = (struct ofp_action_enqueue *)p;
            prev_port = ntohs(ea->port);
            prev_queue = ntohl(ea->queue_id);
            max_len = 0; /* we will not send to the controller anyways - useless */
        } else {
            uint16_t type = ntohs(ah->type);

            if (type < ARRAY_SIZE(of_actions)) {
                execute_ofpat(buffer, key, ah, type);
            } else if (type == OFPAT_VENDOR) {
                execute_vendor(buffer, key, ah);
            }
        }

        p += len;
        actions_len -= len;
    }
    if (prev_port != -1) {
        do_output(dp, buffer, in_port, max_len, prev_port, prev_queue, ignore_no_fwd);
    } else {
        ofpbuf_delete(buffer);
    }
}

static void
do_output(struct datapath *dp, struct ofpbuf *buffer, int in_port,
          size_t max_len, int out_port, uint32_t queue_id,
          bool ignore_no_fwd)
{
    if (out_port != OFPP_CONTROLLER) {
        dp_output_port(dp, buffer, in_port, out_port, queue_id, ignore_no_fwd);
    } else {
        dp_output_control(dp, buffer, in_port, max_len, OFPR_ACTION);
    }
}

/** Takes ownership of 'buffer' and transmits it to 'out_port' on 'dp'.
 */
void
dp_output_port(struct datapath *dp, struct ofpbuf *buffer,
               int in_port, int out_port, uint32_t queue_id,
               bool ignore_no_fwd UNUSED)
{

    assert(buffer);
    switch (out_port) {
    case OFPP_IN_PORT:
        output_packet(dp, buffer, in_port, queue_id);
        break;

    case OFPP_TABLE: {
        struct sw_port *p = dp_lookup_port(dp, in_port);
        if (run_flow_through_tables(dp, buffer, p)) {
            ofpbuf_delete(buffer);
        }
        break;
    }

    case OFPP_FLOOD:
        output_all(dp, buffer, in_port, 1);
        break;

    case OFPP_ALL:
        output_all(dp, buffer, in_port, 0);
        break;

    case OFPP_CONTROLLER:
        dp_output_control(dp, buffer, in_port, UINT16_MAX, OFPR_ACTION);
        break;

    case OFPP_LOCAL:
    default:
        if (in_port == out_port) {
            VLOG_DBG_RL(&rl, "can't directly forward to input port");
            return;
        }
        output_packet(dp, buffer, out_port, queue_id);
        break;
    }
}

/* Takes ownership of 'buffer' and transmits it to 'dp''s controller.  If the
 * packet can be saved in a buffer, then only the first max_len bytes of
 * 'buffer' are sent; otherwise, all of 'buffer' is sent.  'reason' indicates
 * why 'buffer' is being sent. 'max_len' sets the maximum number of bytes that
 * the caller wants to be sent. */
void
dp_output_control(struct datapath *dp, struct ofpbuf *buffer, int in_port,
                  size_t max_len, int reason)
{
    struct ofp_packet_in *opi;
    size_t total_len;
    uint32_t buffer_id;

    buffer_id = save_buffer(buffer);
    total_len = buffer->size;
    if (buffer_id != UINT32_MAX && buffer->size > max_len) {
        buffer->size = max_len;
    }

    opi = ofpbuf_push_uninit(buffer, offsetof(struct ofp_packet_in, data));
    opi->header.version = OFP_VERSION;
    opi->header.type    = OFPT_PACKET_IN;
    opi->header.length  = htons(buffer->size);
    opi->header.xid     = htonl(0);
    opi->buffer_id      = htonl(buffer_id);
    opi->total_len      = htons(total_len);
    opi->in_port        = htons(in_port);
    opi->reason         = reason;
    opi->pad            = 0;
    send_openflow_buffer(dp, buffer, NULL);
}

void *
ofpbuf_push_uninit(struct ofpbuf *b, size_t size) 
{
    ofpbuf_prealloc_headroom(b, size);
    b->data = (char*)b->data - size;
    b->size += size;
    return b->data;
}

/* A connection to a secure channel. */
struct remote {
    struct list node;
    struct rconn *rconn;
#define TXQ_LIMIT 128           /* Max number of packets to queue for tx. */
    int n_txq;                  /* Number of packets queued for tx on rconn. */

    /* Support for reliable, multi-message replies to requests.
     *
     * If an incoming request needs to have a reliable reply that might
     * require multiple messages, it can use remote_start_dump() to set up
     * a callback that will be called as buffer space for replies. */
    int (*cb_dump)(struct datapath *, void *aux);
    void (*cb_done)(void *aux);
    void *cb_aux;
};

/* A reliable connection to an OpenFlow switch or controller.
 *
 * See the large comment in rconn.h for more information. */
struct rconn {
    enum state state;
    time_t state_entered;

    struct vconn *vconn;
    char *name;
    bool reliable;

    struct ofp_queue txq;

    int backoff;
    int max_backoff;
    time_t backoff_deadline;
    time_t last_received;
    time_t last_connected;
    unsigned int packets_sent;
    unsigned int seqno;

    /* In S_ACTIVE and S_IDLE, probably_admitted reports whether we believe
     * that the peer has made a (positive) admission control decision on our
     * connection.  If we have not yet been (probably) admitted, then the
     * connection does not reset the timer used for deciding whether the switch
     * should go into fail-open mode.
     *
     * last_admitted reports the last time we believe such a positive admission
     * control decision was made. */
    bool probably_admitted;
    time_t last_admitted;

    /* These values are simply for statistics reporting, not used directly by
     * anything internal to the rconn (or the secchan for that matter). */
    unsigned int packets_received;
    unsigned int n_attempted_connections, n_successful_connections;
    time_t creation_time;
    unsigned long int total_time_connected;

    /* If we can't connect to the peer, it could be for any number of reasons.
     * Usually, one would assume it is because the peer is not running or
     * because the network is partitioned.  But it could also be because the
     * network topology has changed, in which case the upper layer will need to
     * reassess it (in particular, obtain a new IP address via DHCP and find
     * the new location of the controller).  We set this flag when we suspect
     * that this could be the case. */
    bool questionable_connectivity;
    time_t last_questioned;

    /* Throughout this file, "probe" is shorthand for "inactivity probe".
     * When nothing has been received from the peer for a while, we send out
     * an echo request as an inactivity probe packet.  We should receive back
     * a response. */
    int probe_interval;         /* Secs of inactivity before sending probe. */

    /* Messages sent or received are copied to the monitor connections. */
#define MAX_MONITORS 8
    struct vconn *monitors[8];
    size_t n_monitors;

    /* Protocol statistical informaition. */
    struct ofpstat ofps_rcvd;
    struct ofpstat ofps_sent;

    uint32_t idle_echo_xid;
};