Fixed kiro_create_rdma_memory not setting pointers correctly

Fixed a bug in KIRO Server and Client that prevented preemtive "Post Recv"

6 files changed, 27 insertions, 15 deletions

diff --git a/kiro-client.c b/kiro-client.c
index 9fa15ac..41f4ff5 100644
--- a/kiro-client.c
+++ b/kiro-client.c
@@ -34,6 +34,8 @@
 #include "kiro-client.h"
 #include "kiro-rdma.h"
 
+#include <errno.h>
+
 
 /*
  * Definition of 'private' structures and members and macro to access them
@@ -79,7 +81,7 @@ kiro_client_class_init (KiroClientClass *klass)
 
 
 
-int kiro_client_connect (KiroClient *self, char *address, char* port, size_t timeout)
+int kiro_client_connect (KiroClient *self, char *address, char* port)
 {
     KiroClientPrivate *priv = KIRO_CLIENT_GET_PRIVATE(self);
 
@@ -94,9 +96,10 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim
     hints.ai_port_space = RDMA_PS_IB;
     if(rdma_getaddrinfo(address, port, &hints, &res_addrinfo))
     {
-        printf("Failed to resolve Route to server %s:%s\n",address, port);
+        printf("Failed to contruct address information for %s:%s\n",address, port);
         return -1;
     }
+    printf("Address information created.\n");
     
     struct ibv_qp_init_attr qp_attr;
     memset(&qp_attr, 0, sizeof(qp_attr));
@@ -107,12 +110,12 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim
     qp_attr.qp_context = priv->conn;
     qp_attr.sq_sig_all = 1;
     
-    
     if(rdma_create_ep(&(priv->conn), res_addrinfo, NULL, &qp_attr))
     {
-        printf("Endpoint creation failed.\n");
+        printf("Endpoint creation failed with error: %i\n", errno);
         return -1;
     }
+    printf("Route to server resolved.\n");
     
     struct kiro_connection_context *ctx = (struct kiro_connection_context *)calloc(1,sizeof(struct kiro_connection_context));
     if(!ctx)
@@ -144,9 +147,9 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim
     ctx->cf_mr_recv->size = ctx->cf_mr_send->size = sizeof(struct kiro_ctrl_msg);
     priv->conn->context = ctx;
     
-    if(!rdma_post_recv(priv->conn, priv->conn, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr))
+    if(rdma_post_recv(priv->conn, priv->conn, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr))
     {
-        printf("Posting preemtive receive for connection failed.\n");
+        printf("Posting preemtive receive for connection failed with error: %i\n", errno);
         kiro_destroy_connection_context(ctx);
         rdma_destroy_ep(priv->conn);
         return -1;
@@ -181,7 +184,7 @@ int kiro_client_connect (KiroClient *self, char *address, char* port, size_t tim
     //Create TRB, request RDMA from Server, call kiro_client_sync, ???, Profit!
     
     
-    printf("Connect to server.\n");
+    printf("Connected to server.\n");
     return 0;
     
 }
diff --git a/kiro-client.h b/kiro-client.h
index fb1dccb..0f6881c 100644
--- a/kiro-client.h
+++ b/kiro-client.h
@@ -77,7 +77,7 @@ GObject     kiro_client_new                 (void);
 /* client functions */
 
 
-int         kiro_client_connect             (KiroClient *, char *, char*, size_t);
+int         kiro_client_connect             (KiroClient *, char *, char*);
 
 int         kiro_client_sync                (KiroClient *);
 
diff --git a/kiro-rdma.h b/kiro-rdma.h
index ff39136..25040c8 100644
--- a/kiro-rdma.h
+++ b/kiro-rdma.h
@@ -77,7 +77,7 @@ struct kiro_rdma_mem {
 };
 
 
-static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr *mr, void *mem, size_t mem_size, int access)
+static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr **mr, void *mem, size_t mem_size, int access)
 {
     
     if(mem_size == 0)
@@ -97,8 +97,8 @@ static int kiro_register_rdma_memory (struct ibv_pd *pd, struct ibv_mr *mr, void
         return -1;
     }        
     
-    mr = ibv_reg_mr(pd, mem_handle, mem_size, access);
-    if(!mr)
+    *mr = ibv_reg_mr(pd, mem_handle, mem_size, access);
+    if(!(*mr))
     {
         // Memory Registration failed
         printf("Failed to register memory region!\n");
@@ -125,12 +125,16 @@ static struct kiro_rdma_mem* kiro_create_rdma_memory (struct ibv_pd *pd, size_t
         return NULL;
     }
     
-    if(kiro_register_rdma_memory(pd, krm->mr, krm->mem, mem_size, access))
+    if(kiro_register_rdma_memory(pd, &(krm->mr), krm->mem, mem_size, access))
     {
         free(krm);
         return NULL;
     }
     
+    if(!krm->mem)
+        krm->mem = krm->mr->addr;
+    
+    
     return krm;
     
 }
diff --git a/kiro-server.c b/kiro-server.c
index 145922a..119f0c1 100644
--- a/kiro-server.c
+++ b/kiro-server.c
@@ -99,6 +99,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
         printf("Failed to bind to address %s:%s\n",address, port);
         return -1;
     }
+    printf("Bound to address %s:%s\n",address, port);
     
     struct ibv_qp_init_attr qp_attr;
     memset(&qp_attr, 0, sizeof(qp_attr));
@@ -114,6 +115,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
         printf("Endpoint creation failed.\n");
         return -1;
     }
+    printf("Endpoint created.\n");
     
     if(rdma_listen(priv->base, 0))
     {
@@ -121,6 +123,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
         rdma_destroy_ep(priv->base);
         return -1;
     }
+    printf("Enpoint listening.\n");
     
     priv->client = (struct kiro_connection *)calloc(1, sizeof(struct kiro_connection));
     if(!priv->client)
@@ -131,12 +134,14 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
     }
     priv->client->identifier = 0; //First Client
     
+    printf("Waiting for connection request.\n");
     if(rdma_get_request(priv->base, &(priv->client->id)))
     {
         printf("Failure waiting for clienet connection.\n");
         rdma_destroy_ep(priv->base);
         return -1;
     }
+    printf("Connection Request received.\n");
     
     struct kiro_connection_context *ctx = (struct kiro_connection_context *)calloc(1,sizeof(struct kiro_connection_context));
     if(!ctx)
@@ -174,7 +179,7 @@ int kiro_server_start (KiroServer *self, char *address, char *port)
     ctx->cf_mr_recv->size = ctx->cf_mr_send->size = sizeof(struct kiro_ctrl_msg);
     priv->client->id->context = ctx;
     
-    if(!rdma_post_recv(priv->client->id, priv->client, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr))
+    if(rdma_post_recv(priv->client->id, priv->client, ctx->cf_mr_recv->mem, ctx->cf_mr_recv->size, ctx->cf_mr_recv->mr))
     {
         printf("Posting preemtive receive for connection failed.\n");
         kiro_destroy_connection_context(ctx);
diff --git a/test-client.c b/test-client.c
index 1df974c..67dacf7 100644
--- a/test-client.c
+++ b/test-client.c
@@ -7,6 +7,6 @@
 int main(void)
 {
     KiroClient *client = g_object_new(KIRO_TYPE_CLIENT, NULL);
-    kiro_client_connect(client, "127.0.0.1", "60010", 5000);
+    kiro_client_connect(client, "192.168.11.61", "60010", 5000);
     return 0; 
 }
\ No newline at end of file
diff --git a/test-server.c b/test-server.c
index cac62e6..2b08f15 100644
--- a/test-server.c
+++ b/test-server.c
@@ -7,6 +7,6 @@
 int main(void)
 {
     KiroServer *server = g_object_new(KIRO_TYPE_SERVER, NULL);
-    kiro_server_start(server, "127.0.0.1", "60010");
+    kiro_server_start(server, "192.168.11.61", "60010");
     return 0; 
 }
\ No newline at end of file