4. RDMA读操作实现

任务目标

实现一个RDMA读操作（RDMA Read），使本地节点能够直接从远程节点的内存中读取数据到本地缓冲区，全程无需远程CPU参与。本案例使用rdma_cm与ibverbs API完成连接建立、内存注册、地址交换及读操作执行。

环境与依赖

• Linux系统，已安装rdma-core开发库（包含librdmacm与libibverbs）
• 支持RDMA的网卡（如Mellanox ConnectX系列）及对应驱动
• 编译依赖：libibverbs-dev、librdmacm-dev

核心实现

以下代码实现完整的RDMA读操作流程。程序包含server（提供内存）与client（发起读操作）两种角色，通过命令行参数区分。两端建立连接后，server端注册一块内存并暴露其地址，client端通过RDMA Read将数据直接读取到本地缓冲区。

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <rdma/rdma_cma.h>
#include <infiniband/verbs.h>

#define BUFFER_SIZE 4096

struct conn_info {
    uint64_t addr;
    uint32_t rkey;
};

static struct ibv_pd *pd;
static struct ibv_mr *mr;
static char *buf;

static int setup_qp(struct rdma_cm_id *id) {
    struct ibv_qp_init_attr attr = {0};
    attr.qp_type = IBV_QPT_RC;
    attr.cap.max_send_wr = 16;
    attr.cap.max_recv_wr = 16;
    attr.cap.max_send_sge = 1;
    attr.cap.max_recv_sge = 1;
    attr.sq_sig_all = 1;
    return rdma_create_qp(id, pd, &attr);
}

static void post_read(struct rdma_cm_id *id, struct conn_info *remote) {
    struct ibv_sge sge = {0};
    sge.addr = (uint64_t)buf;
    sge.length = BUFFER_SIZE;
    sge.lkey = mr->lkey;

    struct ibv_send_wr wr = {0};
    wr.opcode = IBV_WR_RDMA_READ;
    wr.sg_list = &sge;
    wr.num_sge = 1;
    wr.wr.rdma.remote_addr = remote->addr;
    wr.wr.rdma.rkey = remote->rkey;

    struct ibv_send_wr *bad_wr;
    if (ibv_post_send(id->qp, &wr, &bad_wr)) {
        perror("ibv_post_send");
        exit(1);
    }
}

static void wait_completion(struct rdma_cm_id *id) {
    struct ibv_wc wc;
    int ret;
    do {
        ret = ibv_poll_cq(id->qp->send_cq, 1, &wc);
    } while (ret == 0);
    if (ret < 0 || wc.status != IBV_WC_SUCCESS) {
        fprintf(stderr, "Work completion failed: %s\n",
                ibv_wc_status_str(wc.status));
        exit(1);
    }
}

static void run_server(void) {
    struct rdma_cm_id *listener = NULL, *conn_id = NULL;
    struct rdma_event_channel *ec = rdma_create_event_channel();
    rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP);
    struct sockaddr_in addr = {0};
    addr.sin_family = AF_INET;
    addr.sin_port = htons(7471);
    rdma_bind_addr(listener, (struct sockaddr *)&addr);
    rdma_listen(listener, 1);

    struct rdma_cm_event *event;
    rdma_get_cm_event(ec, &event);  // CONNECT_REQUEST
    rdma_create_id(ec, &conn_id, NULL, RDMA_PS_TCP);
    rdma_accept(conn_id, NULL);
    rdma_ack_cm_event(event);

    pd = ibv_alloc_pd(conn_id->verbs);
    buf = malloc(BUFFER_SIZE);
    strcpy(buf, "Hello from RDMA server memory!");
    mr = ibv_reg_mr(pd, buf, BUFFER_SIZE,
                    IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ);
    setup_qp(conn_id);

    rdma_get_cm_event(ec, &event);  // ESTABLISHED
    rdma_ack_cm_event(event);

    // 发送本地内存信息给client
    struct conn_info info = {0};
    info.addr = (uint64_t)buf;
    info.rkey = mr->rkey;
    rdma_post_send(conn_id, NULL, &info, sizeof(info), NULL, 0);

    rdma_get_cm_event(ec, &event);  // DISCONNECT
    rdma_ack_cm_event(event);

    rdma_disconnect(conn_id);
    rdma_destroy_id(conn_id);
    rdma_destroy_id(listener);
    rdma_destroy_event_channel(ec);
    ibv_dereg_mr(mr);
    ibv_dealloc_pd(pd);
    free(buf);
}

static void run_client(const char *server_ip) {
    struct rdma_cm_id *conn_id = NULL;
    struct rdma_event_channel *ec = rdma_create_event_channel();
    rdma_create_id(ec, &conn_id, NULL, RDMA_PS_TCP);

    struct sockaddr_in addr = {0};
    addr.sin_family = AF_INET;
    addr.sin_port = htons(7471);
    inet_pton(AF_INET, server_ip, &addr.sin_addr);
    rdma_resolve_addr(conn_id, NULL, (struct sockaddr *)&addr, 2000);

    struct rdma_cm_event *event;
    rdma_get_cm_event(ec, &event);  // ADDR_RESOLVED
    rdma_resolve_route(conn_id, 2000);
    rdma_ack_cm_event(event);

    rdma_get_cm_event(ec, &event);  // ROUTE_RESOLVED
    pd = ibv_alloc_pd(conn_id->verbs);
    buf = malloc(BUFFER_SIZE);
    memset(buf, 0, BUFFER_SIZE);
    mr = ibv_reg_mr(pd, buf, BUFFER_SIZE, IBV_ACCESS_LOCAL_WRITE);
    setup_qp(conn_id);
    rdma_connect(conn_id, NULL);
    rdma_ack_cm_event(event);

    rdma_get_cm_event(ec, &event);  // ESTABLISHED
    rdma_ack_cm_event(event);

    // 接收server端内存地址信息
    struct conn_info remote;
    struct ibv_recv_wr rwr = {0};
    struct ibv_sge rsge = {0};
    rsge.addr = (uint64_t)&remote;
    rsge.length = sizeof(remote);
    rsge.lkey = mr->lkey;
    rwr.sg_list = &rsge;
    rwr.num_sge = 1;
    ibv_post_recv(conn_id->qp, &rwr, NULL);

    struct ibv_wc wc;
    while (ibv_poll_cq(conn_id->qp->recv_cq, 1, &wc) == 0);
    if (wc.status != IBV_WC_SUCCESS) {
        fprintf(stderr, "Recv failed\n");
        exit(1);
    }

    // 执行RDMA Read
    post_read(conn_id, &remote);
    wait_completion(conn_id);

    printf("RDMA Read completed. Data received: %s\n", buf);

    rdma_disconnect(conn_id);
    rdma_destroy_id(conn_id);
    rdma_destroy_event_channel(ec);
    ibv_dereg_mr(mr);
    ibv_dealloc_pd(pd);
    free(buf);
}

int main(int argc, char *argv[]) {
    if (argc < 2) {
        fprintf(stderr, "Usage: %s -s | -c <server_ip>\n", argv[0]);
        return 1;
    }
    if (strcmp(argv[1], "-s") == 0) {
        run_server();
    } else if (strcmp(argv[1], "-c") == 0 && argc == 3) {
        run_client(argv[2]);
    } else {
        fprintf(stderr, "Usage: %s -s | -c <server_ip>\n", argv[0]);
        return 1;
    }
    return 0;
}

编译与运行

# 编译
gcc -o rdma_read rdma_read.c -lrdmacm -libverbs

# 在server节点运行
./rdma_read -s

# 在client节点运行（server_ip替换为server端IP）
./rdma_read -c <server_ip>

验证方法

在server端启动程序后，程序会阻塞等待连接。在client端执行后，client程序会通过RDMA Read直接从server内存中读取数据，并在终端打印：

RDMA Read completed. Data received: Hello from RDMA server memory!

若看到上述输出，说明RDMA读操作成功完成——client端读取到的数据与server端初始化内容一致，且整个过程server端CPU未参与数据拷贝。可通过ibv_devinfo确认网卡状态，使用perftest类工具对比验证性能。

关键说明

• server端内存注册时使用IBV_ACCESS_REMOTE_READ权限，这是允许远程读的必要条件
• client端本地缓冲区注册仅需IBV_ACCESS_LOCAL_WRITE，因为RDMA Read是本地写入操作
• 地址交换通过rdma_post_send发送conn_info结构体完成，包含远程内存地址与rkey
• 本实现使用同步事件处理，实际生产环境建议使用异步事件循环或线程池