SO_REUSEPORT: strange kernel balancer behaviour

[trafdev]

It's like shared acceptor FD and N processes:

Listening proc:

bool HttpServer::Listen(unsigned short port, uint listen_backlog) {
    LOG4CXX_TRACE(kLogger, "Listen");
    if ((sockd_acceptor_ = socket(AF_INET, SOCK_STREAM, 0)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "socket");
        return false;
    }
    struct sockaddr_in sa_in;
    memset(&sa_in, 0, sizeof(sa_in));
    sa_in.sin_family = AF_INET;
    sa_in.sin_port = htons(port);
    sa_in.sin_addr.s_addr = htonl(INADDR_ANY);
    int yes = 1;
    if (setsockopt(sockd_acceptor_, SOL_SOCKET, SO_REUSEPORT, &yes, 
sizeof (yes)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "setsockopt");
        return false;
    }
    if (bind(sockd_acceptor_, (const struct sockaddr*)&sa_in, 
sizeof(sa_in)) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "bind");
        return false;
    }
    if (listen(sockd_acceptor_, listen_backlog) == -1) {
        LOG4CXX_ERROR_ERRNO(kLogger, "socket listen");
        return false;
    }
    if (!fcntl_set(sockd_acceptor_, O_NONBLOCK))
        return false;

    sockd_acceptor_watcher_.set<HttpServer, &HttpServer::AcceptConnCb> 
(this);
    sockd_acceptor_watcher_.start(sockd_acceptor_, ev::READ);

    int wrk = thread::hardware_concurrency();
    while (--wrk) {
        pid_t pid = fork();
        if (pid == 0) {
            LOG4CXX_INFO(kLogger, "process " << wrk <<  " started");
            ev::default_loop().post_fork();
            ev::default_loop().run();
            return true;
        }
    }

    ev::default_loop().run();
    return true;
}


Accept conn callback:

void HttpServer::AcceptConnCb(ev::io &w, int revents) {
    LOG4CXX_TRACE(kLogger, "AcceptConnCb");
    CHECK_LIBEV_ERROR
    struct sockaddr_in addr;
    socklen_t addr_len = sizeof (addr);
    int sockd;
    if ((sockd = accept(sockd_acceptor_, (struct sockaddr *) &addr,
            &addr_len)) == -1) {
        if (errno == EAGAIN || errno == EWOULDBLOCK) {
            return;
        } else {
            LOG4CXX_ERROR_ERRNO(kLogger, "accept");
            return;
        }
    }
    if (!fcntl_set(sockd, O_NONBLOCK))
        return;
    char str[INET_ADDRSTRLEN];
    if (!inet_ntop(AF_INET, &(addr.sin_addr), str, INET_ADDRSTRLEN)) {
        LOG4CXX_ERROR(kLogger, "inet_ntop");
        return;
    }
    // Internal logic follows... ConnIn* c = 
ConnPool::Instance().Add(sockd, str);
    // c->Recv(this);
    return;
}

Accept conn callback is called in N processes on each connection, only 
one wins,
others exit by errno == EAGAIN case. Overhead is almost zero.
Problem is that "wins" distribution is far from equal.


On Mon Jul 22 23:37:57 2013, Adrian Chadd wrote:
> eg:
>
> * one process, one listen thread, multiple dispatch threads?
> * n processes, one listen FD per process, all listening on the same IP:port?
> * one process, each thread listening on the same IP:port?
> * something else?
>
> Thanks,
>
>
>
> -adrian
>
>