Changeset 8313

Timestamp:

04/29/10 18:09:46 (3 years ago)

Author:

iskra

Message:

Add support for immediate completion on the server side.

Location:

trunk/src/io/bmi/bmi_zoid

Files:

• README (modified) (5 diffs)
• server.c (modified) (9 diffs)

trunk/src/io/bmi/bmi_zoid/README

@@ -17 +17 @@
   needed, but is probably already larger than necessary,
 
-- FIXME ZBMI limits
+- ZBMI_SHM_SIZE_TOTAL (in init.c in ZeptoOS
+  packages/zoid/src/zbmi/implementation/ directory): total size of the shared
+  memory buffer used to exchange bulk data between the ZOID daemon and the BMI
+  server; defaults to 512M.
+
+- ZBMI_SHM_SIZE_UNEXP (in init.c in ZeptoOS
+  packages/zoid/src/zbmi/implementation/ directory): part of the shared
+  memory buffer used for unexpected messages; defaults to 1M.
 
 
@@ -103 +110 @@
 
 The zbmi plugin is mostly stateless so far as the compute node clients are
-concerned.  Specifically, the information on posted, but not immediately
-completed expected message sends/receives is stored exclusively on the
-client side.
+concerned.  Specifically, the information on posted expected message
+sends/receives that were not immediately completed is stored exclusively on
+the client side.
 
 All BMI send routines end up in zoid_post_send_common.  That includes
 unexpected messages and list I/O.  This routine attempts to forward the
 message to the zbmi plugin on the I/O node, using zbmi_send.  For
-unexpected messages, zbmi_send is normally expected to succeed and result
-in an immediate completion; however, if the zbmi plugin is out of memory,
+unexpected messages, zbmi_send should normally succeed and result in an
+immediate completion; however, if the zbmi plugin is out of memory,
 zbmi_send will fail with ENOMEM.  The same failure will occur with expected
 messages if a matching receive has not been posted on the I/O node side by
@@ -119 +126 @@
 resulting in an immediate completion.  zbmi_send normally does not block,
 but if BMI_ZOID_POST_TIMEOUT has been enabled, it can, waiting for a
-matching expected message post from the BMI server side.
+matching expected message post from the BMI server side or for memory to be
+released on the BMI server side so that an unexpected message can be
+stored.
 
 The way zbmi_send is forwarded by ZOID, the data payload is only
@@ -149 +158 @@
 using zbmi_test.  zbmi_test can block on the server for the specified time
 if none of the specified requests is initially ready.  zbmi_test returns
-the number of ready requests; if it is non-zero, then zoid_test_common next
-attempts to satisfy those requests by invoking zbmi_send/zbmi_recv.  Those
-send/recv routines could still fail in spite of a successful test, if there
-is no memory, or if the server-side canceled its matching request; this is
+the number of ready requests; if it is non-zero, then the server side must
+have posted matching sends/receives, so zoid_test_common next attempts to
+satisfy those "ready" requests by invoking zbmi_send/zbmi_recv.  Those
+send/recv routines could still fail in spite of a successful test if there
+is no memory or if the server side canceled its matching request; this is
 recoverable.
 
@@ -190 +200 @@
 retry the allocation after every BMI_memfree.
 
-Expected server-side posts, be it sends or receives, are never completed
-immediately: we send a message descriptor to the zbmi plugin which
-registers it and just sends back a confirmation.  When registering we
-exchange the internal BMI id and the internal ZOID id, since that
-simplifies subsequent testing/canceling.
+Expected server-side posts, be it sends or receives, can be completed
+immediately if a matching client-side post (or test) is waiting when
+server-side post is issued.  Note that "immediately" is used liberally
+here; the server-side post will not return until the buffer has been
+transferred to/from the compute node, which can take some time when the
+ZOID server is under heavy load.  For posts not completed immediately we
+send a message descriptor to the zbmi plugin which registers the message
+and just sends back a confirmation.  When registering we exchange the
+internal BMI id and the internal ZOID id, since that simplifies subsequent
+testing/canceling.
 
 Canceling messages is more complex than on the client side.  Generally, we

trunk/src/io/bmi/bmi_zoid/server.c

@@ -88 +88 @@
 static bmi_method_addr_p get_client_addr(int zoid_addr);
 static int enqueue_no_mem(method_op_p op, bmi_size_t total_size);
-static int send_post_cmd(method_op_p op);
+static int send_post_cmd(method_op_p op, int not_immediate, int* length);
 
 
@@ -272 +272 @@
             free(desc);
 
-            if ((op->error_code = -send_post_cmd(op)))
+            if ((op->error_code = -send_post_cmd(op, 1, NULL)))
             {
                 gen_mutex_lock(&error_ops_mutex);
@@ -382 +382 @@
 {
     method_op_p new_op;
+    int ret;
 
     /* Server-side sends are never immediate, so we start by allocating a
@@ -446 +447 @@
     }
 
-    return send_post_cmd(new_op);
+    if ((ret = send_post_cmd(new_op, 0, NULL)) == 1)
+    {
+        /* Immediate completion.  */
+        if (buffer_type == BMI_EXT_ALLOC)
+            BMI_zoid_server_memfree(METHOD_DATA(new_op)->tmp_buffer);
+
+        bmi_dealloc_method_op(new_op);
+    }
+
+    return ret;
 }
 
@@ -460 +470 @@
 {
     method_op_p new_op;
+    int ret, length;
 
     /* Server-side receives are never immediate, so we start by allocating a
@@ -516 +527 @@
     }
 
-    return send_post_cmd(new_op);
+    if ((ret = send_post_cmd(new_op, 0, &length)) == 1)
+    {
+        /* Immediate completion.  */
+        *total_actual_size = length;
+
+        if (buffer_type == BMI_EXT_ALLOC)
+        {
+            /* Copy the memory back to the user buffer(s).  */
+            int j, size_remaining = length;
+            void *buf_cur = METHOD_DATA(new_op)->tmp_buffer;
+            j = 0;
+            while (size_remaining > 0)
+            {
+                int tocopy = (new_op->size_list[j] < size_remaining ?
+                              new_op->size_list[j] : size_remaining);
+
+                memcpy(new_op->buffer_list[j], buf_cur, tocopy);
+                buf_cur += tocopy;
+                size_remaining -= tocopy;
+                j++;
+            }
+
+            BMI_zoid_server_memfree(METHOD_DATA(new_op)->tmp_buffer);
+        }
+
+        bmi_dealloc_method_op(new_op);
+    }
+
+    return ret;
 }
 
@@ -1034 +1073 @@
 }
 
-/* A common internal posting routine for send and receive requests.  */
+/* A common internal posting routine for send and receive requests.
+   "not_immediate" is used for messages triggered from memfree; it is
+   inconvenient at that point for messages to succeed immediately (we
+   would need a separate queue for them).
+   The function returns 0 if posted successfully, 1 for immediate
+   completion, and a negative value if failed.
+   For immediate completions of receives, the length of the received
+   message is stored in *length;
+*/
 static int
-send_post_cmd(method_op_p op)
+send_post_cmd(method_op_p op, int not_immediate, int* length)
 {
     mqd_t reply_queue;
@@ -1059 +1106 @@
            ZBMI_CONTROL_POST_RECV);
     cmd->queue_id = queue_id;
+    cmd->not_immediate = not_immediate;
     cmd->bmi_id = op->op_id;
     cmd->buf.addr = ((struct zoid_addr*)op->addr->method_data)->pid;
@@ -1107 +1155 @@
         return -BMI_ENOMEM;
     }
+    else if (resp.zoid_id == -1)
+    {
+        /* Immediate completion.  */
+        assert(!not_immediate);
+        gen_mutex_unlock(&METHOD_DATA(op)->post_mutex);
+        if (length)
+            *length = resp.length;
+        return 1;
+    }
 
     METHOD_DATA(op)->zoid_buf_id = resp.zoid_id;