Changeset 7861

Timestamp:

07/07/09 18:06:08 (4 years ago)

Author:

sson

Message:

Fixed the bugs when there are more than 4 servers performing stripe alignment and allreduce operations.

Location:

branches/as-branch/src/server

Files:

• allreduce.sm (modified) (1 diff)
• kmeans.sm (modified) (1 diff)
• pipeline.sm (modified) (8 diffs)
• pvfs2-server.h (modified) (1 diff)

branches/as-branch/src/server/allreduce.sm

@@ -175 +175 @@
 
         send_recv_op->u.send_recv.myRank = s_op->u.allreduce.myRank;
-        send_recv_op->u.send_recv.mask = 0x1;
+        send_recv_op->u.send_recv.mask = s_op->u.allreduce.mask;
 
         ret = PINT_sm_push_frame(smcb, 0, send_recv_op);

branches/as-branch/src/server/kmeans.sm

@@ -302 +302 @@
                          s_op->u.kmeans.newClusters[i][j]);
 
-    gossip_debug(GOSSIP_IO_DEBUG, "before Allreduce\n");
+    gossip_debug(GOSSIP_IO_DEBUG, "before Allreduce: delta=%f\n", s_op->u.kmeans.delta);
     for (i=0; i<s_op->u.kmeans.numClusters; i++)
         for (j=0; j<s_op->u.kmeans.numCoords; j++)

branches/as-branch/src/server/pipeline.sm

@@ -312 +312 @@
 {
     struct PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
-    js_p->error_code = 0;
     PVFS_size file_req_offset = s_op->u.pipeline.file_req_offset;
     PINT_request_file_data fdata = s_op->u.pipeline.file_data;
@@ -319 +318 @@
     PVFS_size count;
     PVFS_offset strip_boundary;
-
     PVFS_offset loff = fdata.dist->methods->physical_to_logical_offset(fdata.dist->params, &fdata, s_op->u.pipeline.offsets[0]);
+    js_p->error_code = 0;
     
     s_op->u.pipeline.loff = loff;
@@ -346 +345 @@
         count = (PVFS_size)(s_op->u.pipeline.buffer_used)/((*PVFS_FLOAT).ub);
         if(s_op->u.pipeline.op == 0x5800000f) { /* KMEANS */
-            count = (PVFS_size)(s_op->u.pipeline.buffer_used)/((*PVFS_FLOAT).ub)/18; /* FIXME: 18 is numCoords */
-            s_op->u.pipeline.unaligned_size = 0; /* FIXME */
+            int cunit = ((*PVFS_FLOAT).ub)*18;
+            PVFS_size new_size;
+            float ctmp = ((float)(s_op->u.pipeline.buffer_used-file_req_offset))/((*PVFS_FLOAT).ub)/18;
+
+            gossip_debug(GOSSIP_IO_DEBUG, "ctmp=%f\n", ctmp);
+
+            /* last stripe? */
+            if(s_op->u.pipeline.dfile_index == (s_op->u.pipeline.dfile_count-1))
+                count = floor(ctmp);
+            else
+                count = ceil(ctmp);
+            new_size = count*cunit;
+            s_op->u.pipeline.new_buffer_used = new_size;
+            s_op->u.pipeline.buffer= realloc(s_op->u.pipeline.buffer, new_size);
+            s_op->u.pipeline.unaligned_size = new_size-s_op->u.pipeline.buffer_used;
             gossip_debug(GOSSIP_IO_DEBUG, "FLOAT: count=%lld\n", lld(count));
         }
         gossip_debug(GOSSIP_IO_DEBUG, "strip_size=%ld, count=%lld\n", 
                      dparam->strip_size, lld(count));
-        //strip_boundary = ((int)(loff/(dparam->strip_size)))*(dparam->strip_size);
-        //s_op->u.pipeline.unaligned_size = loff-strip_boundary;
+#if 0
+        strip_boundary = ((int)(loff/(dparam->strip_size)))*(dparam->strip_size);
+        s_op->u.pipeline.unaligned_size = loff-strip_boundary;
         if(dparam->strip_size > (count*(*PVFS_FLOAT).ub)) {
             s_op->u.pipeline.unaligned_size = dparam->strip_size - 
@@ -361 +374 @@
                     - (count*((*PVFS_FLOAT).ub)*18);
         }
+#endif
 #if 0   
         if (loff == strip_boundary && file_req_offset != 0) {
@@ -367 +381 @@
 #endif
         if (s_op->u.pipeline.unaligned_size != 0 && count != 0) {
-            js_p->error_code = UNALIGNED;
+            /* if it is the last stripe, no need to bring data */
+            /* FIXME: we assume no round-robin stripes for now, 
+               but shound't it be more general? */
+            if(s_op->u.pipeline.dfile_index != (s_op->u.pipeline.dfile_count-1))
+                js_p->error_code = UNALIGNED;
             gossip_debug(GOSSIP_IO_DEBUG, "unaligned_size=%lld\n", 
                          lld(s_op->u.pipeline.unaligned_size));
@@ -405 +423 @@
     int next_server_index;
     PVFS_handle next_server_handle;
+    PINT_request_file_data fdata = s_op->u.pipeline.file_data;
+    PVFS_simple_stripe_params *dparam = 
+        (PVFS_simple_stripe_params*)fdata.dist->params;
 
     /* init msgpair */
@@ -426 +447 @@
                                   PVFS_BYTE, &s_op->u.pipeline.file_req);
     
-    s_op->u.pipeline.file_req_offset = (((int)(s_op->u.pipeline.loff/262144))+1)*262144; /* FIXME */
-    
+    /* strip_size from dparam */
+    s_op->u.pipeline.file_req_offset = (((int)(s_op->u.pipeline.loff/dparam->strip_size))+1)*dparam->strip_size;
+
     regions = 1;
     gossip_debug(GOSSIP_IO_DEBUG, "s_op->u.pipeline.file_req_offset=%lld\n", lld(s_op->u.pipeline.file_req_offset));
@@ -788 +810 @@
     kmeans_op->u.kmeans.threshold = 0.001; /* FIXME */
     kmeans_op->u.kmeans.totalNumObjs = 17695; /* FIXME: should be obtained from allreduce(numObjs, SUM) */
-    kmeans_op->u.kmeans.numObjs = s_op->u.pipeline.buffer_used/sizeof(float)/numCoords;
+    kmeans_op->u.kmeans.numObjs = s_op->u.pipeline.new_buffer_used/sizeof(float)/numCoords;
     gossip_debug(GOSSIP_IO_DEBUG, "numObjs=%d\n", kmeans_op->u.kmeans.numObjs);
 

branches/as-branch/src/server/pvfs2-server.h

@@ -386 +386 @@
     char *buffer; 
     PVFS_size buffer_size;
-    PVFS_size buffer_used; 
+    PVFS_size buffer_used;
+    PVFS_size new_buffer_used;
     PVFS_size out_size;
     PINT_segpool_handle_t seg_handle;