Changeset 9336

Timestamp:

06/15/12 12:02:56 (11 months ago)

Author:

ligon

Message:

New Feature: users can now specify at configure time how they want the kernel module to handle the file pointer when an error occurs. The default behavior is to leave the file pointer alone, i.e., the file pointer will always represent the position of the last byte written, even though the user's entire request is not totally satisfied. In this case, a the actual number of bytes written is returned to the user. The new feature will change this default behavior such that the file pointer is repositioned to the byte just prior to the user's write request, a -1 is returned to the user's call, and errno is appropriately set. To enable this feature, add --enable-reset-file-pos on the configure command line when your OrangeFS system is configured.

Removed compiler warnings.

M Makefile.in
M configure.in
M ChangeLog?
M src/kernel/linux-2.6/Makefile.in
M src/kernel/linux-2.6/file.c
M src/kernel/linux-2.6/pvfs2-kernel.h
M src/io/trove/trove-dbpf/dbpf-bstream-direct.c
M src/common/misc/realpath.c
M src/client/usrint/posix.c
M src/client/usrint/stdio.c
M src/client/sysint/client-state-machine.c

Location:

branches/stable

Files:

• ChangeLog (modified) (1 diff)
• Makefile.in (modified) (2 diffs)
• configure.in (modified) (2 diffs)
• src/client/sysint/client-state-machine.c (modified) (1 diff)
• src/client/usrint/posix.c (modified) (3 diffs)
• src/client/usrint/stdio.c (modified) (4 diffs)
• src/common/misc/realpath.c (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-bstream-direct.c (modified) (1 diff)
• src/kernel/linux-2.6/Makefile.in (modified) (1 diff)
• src/kernel/linux-2.6/file.c (modified) (21 diffs)
• src/kernel/linux-2.6/pvfs2-kernel.h (modified) (1 diff)

branches/stable/ChangeLog

@@ -6 +6 @@
 
 Stable
+
+* New Feature: users can now specify at configure time how they want the
+* kernel module to handle the file pointer when an error occurs.  The
+* default behavior is to leave the file pointer alone, i.e., the file
+* pointer will always represent the position of the last byte written,
+* even though the user's entire request is not totally satisfied. In 
+* this case, a the actual number of bytes written is returned to the user.
+* The new feature will change this default behavior such that the file pointer
+* is repositioned to the byte just prior to the user's write request, a
+* -1 is returned to the user's call, and errno is appropriately set.  To
+* enable this feature, add --enable-reset-file-pos on the configure
+* command line when your OrangeFS system is configured.
 
 * Bug fix:  modified pvfs2_readdir, which is executed by the kernel module

branches/stable/Makefile.in

@@ -204 +204 @@
 ULIBDEPLIBS := -lpvfs2
 MMAP_RA_CACHE = @MMAP_RA_CACHE@
+RESET_FILE_POS = @RESET_FILE_POS@
 TRUSTED_CONNECTIONS = @TRUSTED_CONNECTIONS@
 REDHAT_RELEASE = @REDHAT_RELEASE@
@@ -433 +434 @@
 ifdef MMAP_RA_CACHE
 CFLAGS += @MMAP_RA_CACHE@
+endif
+
+# reset the file position pointer when a write call encounters errors (kernel only)
+# by default, this feature is disabled.  Default behavior is to increment the file
+# position pointer as bytes are written.
+ifdef RESET_FILE_POS
+CFLAGS += @RESET_FILE_POS@
 endif
 

branches/stable/configure.in

@@ -298 +298 @@
 ,)
 AC_SUBST(MMAP_RA_CACHE)
+
+dnl a mechanism that resets the file position pointer when an error occurs whether
+dnl or not any bytes were written (kernel interface only).
+RESET_FILE_POS=""
+AC_ARG_ENABLE(reset-file-pos,
+[  --enable-reset-file-pos Resets file position pointer in kernel interface upon error],
+[if test "x$enableval" = "xyes" ; then
+RESET_FILE_POS="-DRESET_FILE_POS"
+fi]
+,)
+AC_SUBST(RESET_FILE_POS)
+
 
 dnl See if the --enable-trusted-connections  option was given to configure
@@ -1503 +1515 @@
 fi
 
+if test "x$RESET_FILE_POS" = "x" -a "x$BUILD_BMI_ONLY" != "x1"; then
+   AC_MSG_RESULT([PVFS2 configured for resetting file position      :  no])
+else
+   AC_MSG_RESULT([PVFS2 configured for resetting file position      : yes])
+fi
+
 if test "x$REDHAT_RELEASE" = "x" -a "x$BUILD_BMI_ONLY" != "x1"; then
    AC_MSG_RESULT([PVFS2 will use workaround for redhat 2.4 kernels  :  no])

branches/stable/src/client/sysint/client-state-machine.c

@@ -531 +531 @@
 
     gossip_debug(GOSSIP_CLIENT_DEBUG,
+            "PINT_client_io_cancel id %lld\n",lld(id));
+    gossip_debug(GOSSIP_CANCEL_DEBUG,
             "PINT_client_io_cancel id %lld\n",lld(id));
 

branches/stable/src/client/usrint/posix.c

@@ -18 +18 @@
 #include "posix-pvfs.h"
 #include "openfile-util.h"
+
+/**
+ * function prototypes not defined in libc, though it is a linux
+ * system call and we define it in the usr lib
+ */
+
+int getdents(unsigned int, struct dirent *, unsigned int);
+int getdents64(unsigned int, struct dirent64 *, unsigned int);
+int flock(int, int);
+int fadvise64(int, off64_t, off64_t, int);
 
 /*
@@ -1305 +1315 @@
 }
 
+
 /**
  * man page calls last arg count but is ambiguous if it is number
@@ -1407 +1418 @@
     return rc;
 }
+
 
 int flock(int fd, int op)

branches/stable/src/client/usrint/stdio.c

@@ -567 +567 @@
 #endif
             /* write data directly */
-            rc = write(stream->_fileno, ptr + rsz_buf, rsz_extra);
+            rc = write(stream->_fileno, (char *)ptr + rsz_buf, rsz_extra);
             if (rc == -1)
             {
@@ -577 +577 @@
         else
         {
-            memcpy(stream->_IO_write_ptr, ptr + rsz_buf, rsz_extra);
+            memcpy(stream->_IO_write_ptr, (char *)ptr + rsz_buf, rsz_extra);
             stream->_IO_write_ptr += rsz_extra;
         }
@@ -717 +717 @@
     }
 
-    /* if more bytes requested */
+   /* if more bytes requested */
     if (rsz_extra)
     {
@@ -727 +727 @@
             {
                 /* read directly from file for remainder of request */
-                bytes_read = read(stream->_fileno, ptr+rsz_buf, rsz_extra);
+                bytes_read = read(stream->_fileno, (char *)ptr+rsz_buf, rsz_extra);
                 if (bytes_read == -1)
                 {

branches/stable/src/common/misc/realpath.c

@@ -43 +43 @@
 
 #define MAX_READLINKS 32
-
-extern int PVFS_util_resolve_absolute(
-    const char* local_path,
-    PVFS_fs_id* out_fs_id,
-    char* out_fs_path,
-    int out_fs_path_max);
 
 #ifdef WIN32

branches/stable/src/io/trove/trove-dbpf/dbpf-bstream-direct.c

@@ -582 +582 @@
 #ifdef HAVE_OPEN_O_DIRECT
     if(!(fcntl(fd, F_GETFL) & O_DIRECT))
-#elif HAVE_FNCTL_F_NOCACHE
+#elif defined(HAVE_FNCTL_F_NOCACHE)
     if (!(fcntl(fd, F_GETFL) & F_NOCACHE)) 
 #else

branches/stable/src/kernel/linux-2.6/Makefile.in

@@ -79 +79 @@
 
 EXTRA_CFLAGS += @MMAP_RA_CACHE@
+EXTRA_CFLAGS += @RESET_FILE_POS@
 EXTRA_CFLAGS += -DPVFS2_VERSION="\"@PVFS2_VERSION@\""
 

branches/stable/src/kernel/linux-2.6/file.c

@@ -50 +50 @@
                             unsigned long xtnr_segs,
                             size_t total_size);
+#ifdef RESET_FILE_POS  
+static ssize_t do_readv_writev_wrapper( struct rw_options *rw);
+#endif
 
 #define wake_up_daemon_for_return(op)             \
@@ -176 +179 @@
         struct {
             loff_t        *offset;
+            loff_t         offset_before_request;
         } io;
         /* Non-contiguous file I/O operations use a vector of offsets */
@@ -451 +455 @@
     {
         gossip_lerr("invalid parameters (rw: %p, vec: %p, nr_segs: %lu, "
-                "total_size: %zd)\n", rw, vec, nr_segs, total_size);
+                    "total_size: %zd)\n", rw, vec, nr_segs, total_size);
         ret = -EINVAL;
         goto out;
@@ -477 +481 @@
         goto out;
     }
-    gossip_debug(GOSSIP_FILE_DEBUG, "GET op %p -> buffer_index %d\n", new_op, buffer_index);
+    gossip_debug(GOSSIP_FILE_DEBUG, "%s/%s(%llu): GET op %p -> buffer_index %d\n"
+                                  , __func__
+                                  ,rw->fnstr
+                                  , llu(rw->pvfs2_inode->refn.handle)
+                                  , new_op, buffer_index);
 
     new_op->uses_shared_memory = 1;
@@ -484 +492 @@
     new_op->upcall.req.io.offset = *(rw->off.io.offset);
 
-    gossip_debug(GOSSIP_FILE_DEBUG, "%s: copy_to_user %d nr_segs %lu, "
-            "offset: %llu total_size: %zd\n", rw->fnstr, rw->copy_to_user_addresses, 
-            nr_segs, llu(*(rw->off.io.offset)), total_size);
+    gossip_debug(GOSSIP_FILE_DEBUG, "%s/%s(%llu): copy_to_user %d nr_segs %lu, "
+                                    "offset: %llu total_size: %zd\n"
+                                   ,__func__
+                                   ,rw->fnstr
+                                   ,llu(rw->pvfs2_inode->refn.handle)
+                                   ,rw->copy_to_user_addresses
+                                   ,nr_segs
+                                   ,llu(*(rw->off.io.offset))
+                                   ,total_size);
+
 
     /* Stage 1: copy the buffers into client-core's address space */
+    /* precopy_buffers only pertains to writes.                   */
     if ((ret = precopy_buffers(buffer_index, rw, vec, nr_segs, total_size)) < 0) 
     {
         goto out;
     }
+
+    gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): Calling post_io_request with tag(%d)\n"
+                                  ,__func__
+                                  ,rw->fnstr
+                                  ,llu(rw->pvfs2_inode->refn.handle)
+                                  ,(int)new_op->tag);
 
     /* Stage 2: Service the I/O operation */
@@ -500 +522 @@
     /* If service_operation() returns -EAGAIN #and# the operation was purged from
      * pvfs2_request_list or htable_ops_in_progress, then we know that the
-     * client was restarted, causing the share memory area to be wiped clean.  To restart an 
-     * I/O operation in this case, we must re-copy the data from the user's iovec 
-     * to a NEW shared memory location.
+     * client was restarted, causing the shared memory area to be wiped clean.  To restart a 
+     * write operation in this case, we must re-copy the data from the user's iovec 
+     * to a NEW shared memory location. To restart a read operation, we must get a new
+     * shared memory location.
     */
     if ( ret == -EAGAIN && op_state_purged(new_op) )
     {
-       gossip_debug(GOSSIP_WAIT_DEBUG,"%s:going to populate_shared_memory.\n",__func__);
+       gossip_debug(GOSSIP_WAIT_DEBUG,"%s:going to repopulate_shared_memory.\n",__func__);
        goto populate_shared_memory;
     }
@@ -540 +563 @@
           goto out;
     }
+
     /* Stage 3: Post copy buffers from client-core's address space */
+    /* postcopy_buffers only pertains to reads.                    */
     if ((ret = postcopy_buffers(buffer_index, rw, vec, nr_segs, 
                     new_op->downcall.resp.io.amt_complete)) < 0) {
@@ -550 +575 @@
         goto out;
     }
+
+    gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): Amount written as returned by the sys-io call:%d\n"
+                                  ,__func__
+                                  ,rw->fnstr
+                                  ,llu(rw->pvfs2_inode->refn.handle)
+                                  ,(int)new_op->downcall.resp.io.amt_complete);
+
     ret = new_op->downcall.resp.io.amt_complete;
-    gossip_debug(GOSSIP_FILE_DEBUG, "wait_for_io returning %ld\n", (long) ret);
+    
     /*
       tell the device file owner waiting on I/O that this read has
@@ -564 +596 @@
     {
         pvfs_bufmap_put(buffer_index);
-        gossip_debug(GOSSIP_FILE_DEBUG, "PUT buffer_index %d\n", buffer_index);
+        gossip_debug(GOSSIP_FILE_DEBUG, "%s(%llu): PUT buffer_index %d\n"
+                                      , rw->fnstr
+                                      , llu(rw->pvfs2_inode->refn.handle)
+                                      , buffer_index);
         buffer_index = -1;
     }
@@ -1288 +1323 @@
         goto out;
     }
+
+    gossip_debug(GOSSIP_FILE_DEBUG,"%s-BEGIN/%s(%llu): count(%d) after estimate_max_iovecs.\n"
+                                  ,__func__
+                                  ,rw->fnstr
+                                  ,llu(pvfs2_inode->refn.handle)
+                                  ,(int)count);
+
     if (rw->type == IO_WRITEV)
     {
@@ -1312 +1354 @@
             goto out;
         }
-        gossip_debug(GOSSIP_FILE_DEBUG, "%s: proceeding with offset : %llu, "
-                                        "size %zd\n",
-                                        rw->fnstr, llu(*offset), count);
-    }
+
+        gossip_debug(GOSSIP_FILE_DEBUG, "%s/%s(%llu): proceeding with offset : %llu, size %d\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,llu(*offset), (int)count);
+    } /*endif IO_WRITEV*/
+
     if (count == 0)
     {
@@ -1328 +1374 @@
      * such that no iovec description straddles a block size limit
      */
+
+    gossip_debug(GOSSIP_FILE_DEBUG,"%s: pvfs_bufmap_size:%d\n"
+                                  ,rw->fnstr
+                                  ,pvfs_bufmap_size_query());
+
     if (count > pvfs_bufmap_size_query())
     {
@@ -1372 +1423 @@
     ptr = iovecptr;
 
-    gossip_debug(GOSSIP_FILE_DEBUG, "%s %zd@%llu\n", 
-            rw->fnstr, count, llu(*offset));
-    gossip_debug(GOSSIP_FILE_DEBUG, "%s: new_nr_segs: %lu, seg_count: %lu\n", 
-            rw->fnstr, new_nr_segs, seg_count);
+    gossip_debug(GOSSIP_FILE_DEBUG, "%s/%s(%llu) %d@%llu\n"
+                                  , __func__
+                                  , rw->fnstr
+                                  , llu(pvfs2_inode->refn.handle)
+                                  , (int)count, llu(*offset));
+    gossip_debug(GOSSIP_FILE_DEBUG, "%s/%s(%llu): new_nr_segs: %lu, seg_count: %lu\n"
+                                  , __func__
+                                  , rw->fnstr
+                                  , llu(pvfs2_inode->refn.handle)
+                                  , new_nr_segs, seg_count);
+
 #ifdef PVFS2_KERNEL_DEBUG
     for (seg = 0; seg < new_nr_segs; seg++)
@@ -1414 +1472 @@
         //{
             /* push the I/O directly through to storage */
-     ret = wait_for_direct_io(rw, ptr, seg_array[seg], each_count);
-        //}
+
+        gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): size of each_count(%d)\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,(int)each_count);
+        gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): BEFORE wait_for_io: offset is %d\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,(int)*offset);
+
+        ret = wait_for_direct_io(rw, ptr, seg_array[seg], each_count);
+
+        gossip_debug(GOSSIP_FILE_DEBUG,"%s%s(%llu): return from wait_for_io:%d\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,(int)ret);
+
         if (ret < 0)
         {
             goto out;
         }
+
         /* advance the iovec pointer */
         ptr += seg_array[seg];
@@ -1427 +1504 @@
         amt_complete = ret;
 
+        gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): AFTER wait_for_io: offset is %d\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,(int)*offset);
+
         /* if we got a short I/O operations,
          * fall out and return what we got so far 
@@ -1434 +1517 @@
             break;
         }
-    }
+    }/*end while*/
+
     if (total_count > 0)
     {
@@ -1459 +1543 @@
         mark_inode_dirty_sync(inode);
     }
+
+    gossip_debug(GOSSIP_FILE_DEBUG,"%s/%s(%llu): Value(%d) returned.\n"
+                                      ,__func__
+                                      ,rw->fnstr
+                                      ,llu(pvfs2_inode->refn.handle)
+                                      ,(int)ret);
+
     return ret;
 }
@@ -1529 +1620 @@
     g_pvfs2_stats.reads++;
 
+#ifdef RESET_FILE_POS
+    return do_readv_writev_wrapper(&rw);
+#else
     return do_readv_writev(&rw);
+#endif
 }
 
@@ -1560 +1655 @@
     rw.off.io.offset = offset;
     g_pvfs2_stats.writes++;
+
+#ifdef RESET_FILE_POS
+    return do_readv_writev_wrapper(&rw);
+#else
     return do_readv_writev(&rw);
+#endif
 }
 
@@ -2701 +2801 @@
     if (!rw->async)
     {
+
+#ifdef RESET_FILE_POS
+        error = do_readv_writev_wrapper(rw);
+#else
         error = do_readv_writev(rw);
+#endif
+
         /* not sure this is the correct place or way to update ki_pos but it
          * definitely needs to occur somehow. otherwise, a write following 
@@ -3553 +3659 @@
 #endif
 
+
+#ifdef RESET_FILE_POS
+/* This function wrapper imposes the rule that the user's
+ * request was either entirely fulfilled or it wasn't.  If it wasn't,
+ * then errno will be set appropriately, -1 will be returned as the
+ * request's return value, and the file offset will be repositioned to
+ * the beginning of the request.   If it was successfully completed, then
+ * the amount written/read will be returned and the file offset will be
+ * incremented the appropriate amount.
+ */
+static ssize_t do_readv_writev_wrapper( struct rw_options *rw)
+{
+    ssize_t ret;
+    
+    gossip_err("Wrapper called.\n");
+
+    /* Save the file's current offset before issuing this read/write
+     * request.
+     */
+    rw->off.io.offset_before_request = *(rw->off.io.offset);
+
+    /* If the return code from the request is negative,
+     * restore the offset to it's original value.
+     */
+    ret = do_readv_writev(rw);
+    if (ret < 0)
+    {
+       *(rw->off.io.offset) = rw->off.io.offset_before_request;
+    }
+    return (ret);
+}
+#endif
+
+
+
 /*
  * Local variables:

branches/stable/src/kernel/linux-2.6/pvfs2-kernel.h

@@ -385 +385 @@
     /*  buffer and re-populate it.                                       */ 
     int uses_shared_memory;
-
 
     pvfs2_upcall_t upcall;