diff options
Diffstat (limited to 'src/kiro-rdma.h')
| -rw-r--r-- | src/kiro-rdma.h | 230 |
1 files changed, 230 insertions, 0 deletions
diff --git a/src/kiro-rdma.h b/src/kiro-rdma.h new file mode 100644 index 0000000..fa16fd1 --- /dev/null +++ b/src/kiro-rdma.h @@ -0,0 +1,230 @@ +/* Copyright (C) 2014 Timo Dritschler <timo.dritschler@kit.edu> + (Karlsruhe Institute of Technology) + + This library is free software; you can redistribute it and/or modify it + under the terms of the GNU Lesser General Public License as published by the + Free Software Foundation; either version 2.1 of the License, or (at your + option) any later version. + + This library is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more + details. + + You should have received a copy of the GNU Lesser General Public License along + with this library; if not, write to the Free Software Foundation, Inc., 51 + Franklin St, Fifth Floor, Boston, MA 02110, USA +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#ifndef __KIRO_RDMA_H__ +#define __KIRO_RDMA_H__ + + +#include <rdma/rdma_cma.h> + + +struct kiro_connection_context { + + // Information and necessary structurs + uint32_t identifier; // Unique Identifier for this connection (Application Specific) + struct kiro_rdma_mem *cf_mr_recv; // Control-Flow Memory Region Receive + struct kiro_rdma_mem *cf_mr_send; // Control-Flow Memory Region Send + struct kiro_rdma_mem *rdma_mr; // Memory Region for RDMA Operations + + struct ibv_mr peer_mr; // RDMA Memory Region Information of the peer + + enum { + KIRO_IDLE, + KIRO_MRI_REQUESTED, // Memory Region Information Requested + KIRO_RDMA_ESTABLISHED, // MRI Exchange complete. RDMA is ready + KIRO_RDMA_ACTIVE // RDMA Operation is being performed + } rdma_state; + +}; + + +struct kiro_connection { + + uint32_t identifier; + struct rdma_cm_id *id; + struct kiro_connection *next; + +}; + + +struct kiro_ctrl_msg { + + enum { + KIRO_REQ_RDMA, // Requesting RDMA Access to/from the peer + KIRO_ACK_RDMA, // acknowledge RDMA Request and provide Memory Region Information + KIRO_REJ_RDMA // RDMA Request rejected :( (peer_mri will be invalid) + } msg_type; + + struct ibv_mr peer_mri; + +}; + + +struct kiro_rdma_mem { + + void *mem; // Pointer to the beginning of the memory block + struct ibv_mr *mr; // Memory Region associated with the memory + size_t size; // Size in Bytes of the memory block + +}; + + +static int kiro_attach_qp (struct rdma_cm_id *id) +{ + if(!id) + return -1; + + id->pd = ibv_alloc_pd(id->verbs); + id->send_cq_channel = ibv_create_comp_channel(id->verbs); + id->recv_cq_channel = id->send_cq_channel; //we use one shared completion channel + id->send_cq = ibv_create_cq(id->verbs, 1, id, id->send_cq_channel, 0); + id->recv_cq = id->send_cq; //we use one shared completion queue + + struct ibv_qp_init_attr qp_attr; + memset(&qp_attr, 0, sizeof(struct ibv_qp_init_attr)); + qp_attr.qp_context = (uintptr_t)id; + qp_attr.send_cq = id->send_cq; + qp_attr.recv_cq = id->recv_cq; + qp_attr.qp_type = IBV_QPT_RC; + qp_attr.cap.max_send_wr = 1; + qp_attr.cap.max_recv_wr = 1; + qp_attr.cap.max_send_sge = 1; + qp_attr.cap.max_recv_sge = 1; + + return rdma_create_qp(id, id->pd, &qp_attr); +} + + +static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr **mr, void *mem, size_t mem_size, int access) +{ + + if(mem_size == 0) + { + printf("Cant allocate memory of size '0'.\n"); + return -1; + } + + void *mem_handle = mem; + + if(!mem_handle) + mem_handle = malloc(mem_size); + + if(!mem_handle) + { + printf("Failed to allocate memory [Register Memory]."); + return -1; + } + + *mr = ibv_reg_mr(pd, mem_handle, mem_size, access); + if(!(*mr)) + { + // Memory Registration failed + printf("Failed to register memory region!\n"); + free(mem_handle); + return -1; + } + + return 0; +} + + +static struct kiro_rdma_mem* kiro_create_rdma_memory (struct ibv_pd *pd, size_t mem_size, int access) +{ + if(mem_size == 0) + { + printf("Cant allocate memory of size '0'.\n"); + return NULL; + } + + struct kiro_rdma_mem *krm = (struct kiro_rdma_mem *)calloc(1, sizeof(struct kiro_rdma_mem)); + if(!krm) + { + printf("Failed to create new KIRO RDMA Memory.\n"); + return NULL; + } + + if(kiro_register_rdma_memory(pd, &(krm->mr), krm->mem, mem_size, access)) + { + free(krm); + return NULL; + } + + if(!krm->mem) + krm->mem = krm->mr->addr; + + + return krm; + +} + + +static void kiro_destroy_rdma_memory (struct kiro_rdma_mem *krm) +{ + if(!krm) + return; + + if(krm->mr) + ibv_dereg_mr(krm->mr); + + if(krm->mem) + free(krm->mem); + + free(krm); + krm = NULL; +} + + +static void kiro_destroy_connection_context (struct kiro_connection_context **ctx) +{ + if(!ctx) + return; + + if(!(*ctx)) + return; + + if((*ctx)->cf_mr_recv) + kiro_destroy_rdma_memory((*ctx)->cf_mr_recv); + if((*ctx)->cf_mr_send) + kiro_destroy_rdma_memory((*ctx)->cf_mr_send); + + //The RDMA-Memory Region normally contains allocated memory from the USER that has + //just been 'registered' for RDMA. DON'T free it! Just deregister it. The user is + //responsible for freeing this memory. + if((*ctx)->rdma_mr) + { + if((*ctx)->rdma_mr->mr) + ibv_dereg_mr((*ctx)->rdma_mr->mr); + + free((*ctx)->rdma_mr); + (*ctx)->rdma_mr = NULL; + } + + free(*ctx); + *ctx = NULL; +} + + +static void kiro_destroy_connection (struct rdma_cm_id **conn) +{ + if(!(*conn)) + return; + + rdma_disconnect(*conn); + struct kiro_connection_context *ctx = (struct kiro_connection_context *)((*conn)->context); + if(ctx) + kiro_destroy_connection_context(&ctx); + + rdma_destroy_ep(*conn); + *conn = NULL; +} + + +#endif //__KIRO_RDMA_H__
\ No newline at end of file |