Changeset 8317

Timestamp:

04/30/10 16:00:57 (3 years ago)

Author:

bligon

Message:

merge of B2O-Blue-Sync,HEAD,and latest of Orange.

Location:

branches/Orange-Branch

Files:

• Makefile.in (modified) (2 diffs)
• configure (modified) (13 diffs)
• configure.in (modified) (4 diffs)
• doc/REFERENCES.bib (modified) (6 diffs)
• doc/add-server-req (modified) (1 diff)
• doc/pvfs2-faq.tex (modified) (2 diffs)
• doc/pvfs2-logging.txt (modified) (1 diff)
• doc/pvfs2-quickstart.tex (modified) (1 diff)
• examples/heartbeat/hardware-specific/Filesystem-qla-monitor (modified) (3 diffs)
• include/pvfs2-debug.h (modified) (1 diff)
• include/pvfs2-encode-stubs.h (modified) (1 diff)
• include/pvfs2-mgmt.h (modified) (3 diffs)
• include/pvfs2-types.h (modified) (2 diffs)
• maint/config/kernel.m4 (modified) (4 diffs)
• pvfs2-config.h.in (modified) (4 diffs)
• src/apps/admin/pvfs2-ls.c (modified) (28 diffs)
• src/apps/admin/pvfs2-stat.c (modified) (3 diffs)
• src/apps/fuse/pvfs2fuse.c (modified) (4 diffs)
• src/apps/kernel/linux/pvfs2-client-core.c (modified) (8 diffs)
• src/apps/kernel/linux/pvfs2-client.c (modified) (1 diff)
• src/client/sysint/client-state-machine.c (modified) (11 diffs)
• src/client/sysint/client-state-machine.h (modified) (2 diffs)
• src/client/sysint/fs-add.sm (modified) (11 diffs)
• src/client/sysint/initialize.c (modified) (1 diff)
• src/client/sysint/server-get-config.sm (modified) (3 diffs)
• src/client/sysint/sys-create.sm (modified) (1 diff)
• src/client/sysint/sys-io.sm (modified) (1 diff)
• src/client/sysint/sys-truncate.sm (modified) (2 diffs)
• src/common/gossip/gossip.c (modified) (1 diff)
• src/common/misc/pint-hint.c (modified) (1 diff)
• src/common/misc/pint-util.c (modified) (3 diffs)
• src/common/misc/pint-util.h (modified) (1 diff)
• src/common/misc/server-config.c (modified) (1 diff)
• src/common/misc/state-machine-fns.c (modified) (1 diff)
• src/io/bmi/bmi-method-support.h (modified) (2 diffs)
• src/io/bmi/bmi.c (modified) (12 diffs)
• src/io/bmi/bmi_gm/bmi-gm.c (modified) (1 diff)
• src/io/bmi/bmi_ib/ib.c (modified) (8 diffs)
• src/io/bmi/bmi_mx/README (modified) (3 diffs)
• src/io/bmi/bmi_mx/mx.c (modified) (62 diffs)
• src/io/bmi/bmi_mx/mx.h (modified) (4 diffs)
• src/io/bmi/bmi_portals/portals.c (modified) (1 diff)
• src/io/bmi/bmi_tcp/socket-collection.c (modified) (1 diff)
• src/io/flow/flow.h (modified) (1 diff)
• src/io/flow/flowproto-bmi-cache/flowproto-bmi-cache-server.c (modified) (2 diffs)
• src/io/flow/flowproto-bmi-trove/flowproto-multiqueue.c (modified) (3 diffs)
• src/io/job/job.c (modified) (4 diffs)
• src/io/trove/pvfs2-storage.h (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-alt-aio.c (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-attr-cache.c (modified) (2 diffs)
• src/io/trove/trove-dbpf/dbpf-bstream-direct.c (modified) (4 diffs)
• src/io/trove/trove-dbpf/dbpf-dspace.c (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-mgmt.c (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-null-aio.c (modified) (1 diff)
• src/io/trove/trove-dbpf/dbpf-open-cache.c (modified) (2 diffs)
• src/io/trove/trove-dbpf/dbpf-open-cache.h (modified) (1 diff)
• src/kernel/linux-2.6/inode.c (modified) (1 diff)
• src/kernel/linux-2.6/namei.c (modified) (2 diffs)
• src/kernel/linux-2.6/pvfs2-proc.c (modified) (7 diffs)
• src/kernel/linux-2.6/super.c (modified) (8 diffs)
• src/proto/PINT-le-bytefield.c (modified) (1 diff)
• src/proto/endecode-funcs.h (modified) (1 diff)
• src/proto/pvfs2-attr.h (modified) (4 diffs)
• src/proto/pvfs2-req-proto.h (modified) (49 diffs)
• src/server/chdirent.sm (modified) (1 diff)
• src/server/crdirent.sm (modified) (1 diff)
• src/server/lookup.sm (modified) (1 diff)
• src/server/module.mk.in (modified) (1 diff)
• src/server/prelude.sm (modified) (1 diff)
• src/server/pvfs2-server.c (modified) (4 diffs)
• src/server/pvfs2-server.h (modified) (1 diff)
• src/server/rmdirent.sm (modified) (1 diff)
• src/server/set-attr.sm (modified) (1 diff)
• src/server/tree-communicate.sm (modified) (17 diffs)
• src/server/unstuff.sm (modified) (1 diff)
• test/automated/README.tests (modified) (1 diff)
• test/automated/vfs-tests.d/ltp (modified) (1 diff)
• test/configure (modified) (4 diffs)
• test/configure.in (modified) (3 diffs)
• test/io/bmi/pingpong.c (modified) (13 diffs)
• test/io/trove/module.mk.in (modified) (1 diff)
• test/io/trove/trove-bench-concurrent.c (modified) (2 diffs)
• test/pvfs2-test-config.h.in (modified) (2 diffs)

branches/Orange-Branch/Makefile.in

@@ -79 +79 @@
 BUILD_OPENIB = @BUILD_OPENIB@
 BUILD_PORTALS = @BUILD_PORTALS@
+BUILD_ZOID = @BUILD_ZOID@
 BUILD_VIS = @BUILD_VIS@
 BUILD_KARMA = @BUILD_KARMA@
@@ -400 +401 @@
 endif
 
+ifdef BUILD_ZOID
+    CFLAGS += -D__STATIC_METHOD_BMI_ZOID__
+endif
+
 # enable mmap-readahead cache (unless disabled by configure)
 ifdef MMAP_RA_CACHE

branches/Orange-Branch/configure

@@ -643 +643 @@
 GOSSIP_ENABLE_BACKTRACE
 SERVER_LDFLAGS
+ZOID_SRCDIR
+BUILD_ZOID
 PORTALS_LIBS
 PORTALS_INCS
@@ -819 +821 @@
 with_portals_includes
 with_portals_libs
+with_zoid
 enable_static_server
 '
@@ -1527 +1530 @@
   --with-portals-libs=<dir>
                           Extra LIBS to link Portals libraries
+  --with-zoid=<dir>         Location of the ZOID tree (default no ZOID)
 
 Some influential environment variables:
@@ -3687 +3691 @@
 
 
+if test "${ac_cv_header_pwd_h+set}" = set; then
+  { $as_echo "$as_me:$LINENO: checking for pwd.h" >&5
+$as_echo_n "checking for pwd.h... " >&6; }
+if test "${ac_cv_header_pwd_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_pwd_h" >&5
+$as_echo "$ac_cv_header_pwd_h" >&6; }
+else
+  # Is the header compilable?
+{ $as_echo "$as_me:$LINENO: checking pwd.h usability" >&5
+$as_echo_n "checking pwd.h usability... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <pwd.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  ac_header_compiler=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        ac_header_compiler=no
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+$as_echo "$ac_header_compiler" >&6; }
+
+# Is the header present?
+{ $as_echo "$as_me:$LINENO: checking pwd.h presence" >&5
+$as_echo_n "checking pwd.h presence... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <pwd.h>
+_ACEOF
+if { (ac_try="$ac_cpp conftest.$ac_ext"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null && {
+         test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       }; then
+  ac_header_preproc=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+
+rm -f conftest.err conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+$as_echo "$ac_header_preproc" >&6; }
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: accepted by the compiler, rejected by the preprocessor!" >&5
+$as_echo "$as_me: WARNING: pwd.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: pwd.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: present but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: pwd.h: present but cannot be compiled" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h:     check for missing prerequisite headers?" >&5
+$as_echo "$as_me: WARNING: pwd.h:     check for missing prerequisite headers?" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: see the Autoconf documentation" >&5
+$as_echo "$as_me: WARNING: pwd.h: see the Autoconf documentation" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h:     section \"Present But Cannot Be Compiled\"" >&5
+$as_echo "$as_me: WARNING: pwd.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: proceeding with the preprocessor's result" >&5
+$as_echo "$as_me: WARNING: pwd.h: proceeding with the preprocessor's result" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: pwd.h: in the future, the compiler will take precedence" >&5
+$as_echo "$as_me: WARNING: pwd.h: in the future, the compiler will take precedence" >&2;}
+
+    ;;
+esac
+{ $as_echo "$as_me:$LINENO: checking for pwd.h" >&5
+$as_echo_n "checking for pwd.h... " >&6; }
+if test "${ac_cv_header_pwd_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+else
+  ac_cv_header_pwd_h=$ac_header_preproc
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_pwd_h" >&5
+$as_echo "$ac_cv_header_pwd_h" >&6; }
+
+fi
+if test "x$ac_cv_header_pwd_h" = x""yes; then
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_GETPWUID 1
+_ACEOF
+
+fi
+
+
 if test "${ac_cv_header_sys_vfs_h+set}" = set; then
   { $as_echo "$as_me:$LINENO: checking for sys/vfs.h" >&5
@@ -10182 +10322 @@
         fi
 
-                { $as_echo "$as_me:$LINENO: checking for 6th argument to sysctl proc handlers" >&5
-$as_echo_n "checking for 6th argument to sysctl proc handlers... " >&6; }
-                cat >conftest.$ac_ext <<_ACEOF
+
+                { $as_echo "$as_me:$LINENO: checking for file argument to sysctl proc handlers" >&5
+$as_echo_n "checking for file argument to sysctl proc handlers... " >&6; }
+        cat >conftest.$ac_ext <<_ACEOF
 /* confdefs.h.  */
 _ACEOF
@@ -10199 +10340 @@
 {
 
-            proc_dointvec_minmax(NULL, 0, NULL, NULL, NULL, NULL);
+                struct ctl_table * ctl = NULL;
+                int write = 0;
+                struct file * filp = NULL;
+                void __user * buffer = NULL;
+                size_t * lenp = NULL;
+                loff_t * ppos = NULL;
+
+                proc_dointvec_minmax(ctl, write, filp, buffer, lenp, ppos);
 
   ;
@@ -10227 +10375 @@
 
 cat >>confdefs.h <<\_ACEOF
-#define HAVE_PROC_HANDLER_SIX_ARG 1
+#define HAVE_PROC_HANDLER_FILE_ARG 1
+_ACEOF
+
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+        { $as_echo "$as_me:$LINENO: checking for ppos argument to sysctl proc handlers" >&5
+$as_echo_n "checking for ppos argument to sysctl proc handlers... " >&6; }
+                cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+            #define __KERNEL__
+            #include <linux/fs.h>
+            #include <linux/sysctl.h>
+
+int
+main ()
+{
+
+                struct ctl_table * ctl = NULL;
+                int write = 0;
+                void __user * buffer = NULL;
+                size_t * lenp = NULL;
+                loff_t * ppos = NULL;
+
+                proc_dointvec_minmax(ctl, write, buffer, lenp, ppos);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  { $as_echo "$as_me:$LINENO: result: yes" >&5
+$as_echo "yes" >&6; }
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_PROC_HANDLER_PPOS_ARG 1
 _ACEOF
 
@@ -12927 +13140 @@
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
 
+        { $as_echo "$as_me:$LINENO: checking for s_dirty in struct super_block" >&5
+$as_echo_n "checking for s_dirty in struct super_block... " >&6; }
+        cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+                #define __KERNEL__
+                #include <linux/fs.h>
+
+int
+main ()
+{
+
+                struct super_block *s;
+                list_empty(&s->s_dirty);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  { $as_echo "$as_me:$LINENO: result: yes" >&5
+$as_echo "yes" >&6; }
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_SB_DIRTY_LIST 1
+_ACEOF
+
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
                         tmp_cflags=$CFLAGS
         CFLAGS="$CFLAGS -Werror"
@@ -12975 +13247 @@
 cat >>confdefs.h <<\_ACEOF
 #define HAVE_CURRENT_FSUID 1
+_ACEOF
+
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+        CFLAGS=$tmp_cflags
+
+                { $as_echo "$as_me:$LINENO: checking if kernel backing_dev_info struct has a name field" >&5
+$as_echo_n "checking if kernel backing_dev_info struct has a name field... " >&6; }
+        cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+                #define __KERNEL__
+                #include <linux/fs.h>
+                #include <linux/backing-dev.h>
+
+int
+main ()
+{
+
+                struct backing_dev_info foo =
+                {
+                    .name = "foo"
+                };
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  { $as_echo "$as_me:$LINENO: result: yes" >&5
+$as_echo "yes" >&6; }
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_BACKING_DEV_INFO_NAME 1
+_ACEOF
+
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        { $as_echo "$as_me:$LINENO: result: no" >&5
+$as_echo "no" >&6; }
+
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+
+                tmp_cflags=$CFLAGS
+        CFLAGS="$CFLAGS -Werror"
+        { $as_echo "$as_me:$LINENO: checking for bdi_init" >&5
+$as_echo_n "checking for bdi_init... " >&6; }
+        cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+
+                #define __KERNEL__
+                #include <linux/fs.h>
+                #include <linux/backing-dev.h>
+
+int
+main ()
+{
+
+                int ret = bdi_init(NULL);
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  { $as_echo "$as_me:$LINENO: result: yes" >&5
+$as_echo "yes" >&6; }
+
+cat >>confdefs.h <<\_ACEOF
+#define HAVE_BDI_INIT 1
 _ACEOF
 
@@ -17490 +17886 @@
 
 
+
+
+# Check whether --with-zoid was given.
+if test "${with_zoid+set}" = set; then
+  withval=$with_zoid; if test -z "$withval" -o "$withval" = yes ; then
+            { { $as_echo "$as_me:$LINENO: error: Option --with-zoid requires the path to your ZOID source tree." >&5
+$as_echo "$as_me: error: Option --with-zoid requires the path to your ZOID source tree." >&2;}
+   { (exit 1); exit 1; }; }
+        elif test "$withval" != no ; then
+            ZOID_SRCDIR="$withval"
+        fi
+
+fi
+
+    if test -n "$ZOID_SRCDIR" ; then
+        save_cppflags="$CPPFLAGS"
+        CPPFLAGS="$CPPFLAGS -Isrc/io/bmi -I$ZOID_SRCDIR/include -I$ZOID_SRCDIR/zbmi -I$ZOID_SRCDIR/zbmi/implementation"
+        if test "${ac_cv_header_zbmi_h+set}" = set; then
+  { $as_echo "$as_me:$LINENO: checking for zbmi.h" >&5
+$as_echo_n "checking for zbmi.h... " >&6; }
+if test "${ac_cv_header_zbmi_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zbmi_h" >&5
+$as_echo "$ac_cv_header_zbmi_h" >&6; }
+else
+  # Is the header compilable?
+{ $as_echo "$as_me:$LINENO: checking zbmi.h usability" >&5
+$as_echo_n "checking zbmi.h usability... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <zbmi.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  ac_header_compiler=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        ac_header_compiler=no
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+$as_echo "$ac_header_compiler" >&6; }
+
+# Is the header present?
+{ $as_echo "$as_me:$LINENO: checking zbmi.h presence" >&5
+$as_echo_n "checking zbmi.h presence... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <zbmi.h>
+_ACEOF
+if { (ac_try="$ac_cpp conftest.$ac_ext"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null && {
+         test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       }; then
+  ac_header_preproc=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+
+rm -f conftest.err conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+$as_echo "$ac_header_preproc" >&6; }
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: accepted by the compiler, rejected by the preprocessor!" >&5
+$as_echo "$as_me: WARNING: zbmi.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: zbmi.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: present but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: zbmi.h: present but cannot be compiled" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h:     check for missing prerequisite headers?" >&5
+$as_echo "$as_me: WARNING: zbmi.h:     check for missing prerequisite headers?" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: see the Autoconf documentation" >&5
+$as_echo "$as_me: WARNING: zbmi.h: see the Autoconf documentation" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h:     section \"Present But Cannot Be Compiled\"" >&5
+$as_echo "$as_me: WARNING: zbmi.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: proceeding with the preprocessor's result" >&5
+$as_echo "$as_me: WARNING: zbmi.h: proceeding with the preprocessor's result" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi.h: in the future, the compiler will take precedence" >&5
+$as_echo "$as_me: WARNING: zbmi.h: in the future, the compiler will take precedence" >&2;}
+
+    ;;
+esac
+{ $as_echo "$as_me:$LINENO: checking for zbmi.h" >&5
+$as_echo_n "checking for zbmi.h... " >&6; }
+if test "${ac_cv_header_zbmi_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+else
+  ac_cv_header_zbmi_h=$ac_header_preproc
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zbmi_h" >&5
+$as_echo "$ac_cv_header_zbmi_h" >&6; }
+
+fi
+if test "x$ac_cv_header_zbmi_h" = x""yes; then
+  :
+else
+  { { $as_echo "$as_me:$LINENO: error: Header zbmi.h not found." >&5
+$as_echo "$as_me: error: Header zbmi.h not found." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+
+        if test "${ac_cv_header_zoid_api_h+set}" = set; then
+  { $as_echo "$as_me:$LINENO: checking for zoid_api.h" >&5
+$as_echo_n "checking for zoid_api.h... " >&6; }
+if test "${ac_cv_header_zoid_api_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zoid_api_h" >&5
+$as_echo "$ac_cv_header_zoid_api_h" >&6; }
+else
+  # Is the header compilable?
+{ $as_echo "$as_me:$LINENO: checking zoid_api.h usability" >&5
+$as_echo_n "checking zoid_api.h usability... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <zoid_api.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  ac_header_compiler=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        ac_header_compiler=no
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+$as_echo "$ac_header_compiler" >&6; }
+
+# Is the header present?
+{ $as_echo "$as_me:$LINENO: checking zoid_api.h presence" >&5
+$as_echo_n "checking zoid_api.h presence... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <zoid_api.h>
+_ACEOF
+if { (ac_try="$ac_cpp conftest.$ac_ext"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null && {
+         test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       }; then
+  ac_header_preproc=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+
+rm -f conftest.err conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+$as_echo "$ac_header_preproc" >&6; }
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: accepted by the compiler, rejected by the preprocessor!" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: present but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: present but cannot be compiled" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h:     check for missing prerequisite headers?" >&5
+$as_echo "$as_me: WARNING: zoid_api.h:     check for missing prerequisite headers?" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: see the Autoconf documentation" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: see the Autoconf documentation" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h:     section \"Present But Cannot Be Compiled\"" >&5
+$as_echo "$as_me: WARNING: zoid_api.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: proceeding with the preprocessor's result" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: proceeding with the preprocessor's result" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zoid_api.h: in the future, the compiler will take precedence" >&5
+$as_echo "$as_me: WARNING: zoid_api.h: in the future, the compiler will take precedence" >&2;}
+
+    ;;
+esac
+{ $as_echo "$as_me:$LINENO: checking for zoid_api.h" >&5
+$as_echo_n "checking for zoid_api.h... " >&6; }
+if test "${ac_cv_header_zoid_api_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+else
+  ac_cv_header_zoid_api_h=$ac_header_preproc
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zoid_api_h" >&5
+$as_echo "$ac_cv_header_zoid_api_h" >&6; }
+
+fi
+if test "x$ac_cv_header_zoid_api_h" = x""yes; then
+  :
+else
+  { { $as_echo "$as_me:$LINENO: error: Header zoid_api.h not found." >&5
+$as_echo "$as_me: error: Header zoid_api.h not found." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+
+        if test "${ac_cv_header_zbmi_protocol_h+set}" = set; then
+  { $as_echo "$as_me:$LINENO: checking for zbmi_protocol.h" >&5
+$as_echo_n "checking for zbmi_protocol.h... " >&6; }
+if test "${ac_cv_header_zbmi_protocol_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zbmi_protocol_h" >&5
+$as_echo "$ac_cv_header_zbmi_protocol_h" >&6; }
+else
+  # Is the header compilable?
+{ $as_echo "$as_me:$LINENO: checking zbmi_protocol.h usability" >&5
+$as_echo_n "checking zbmi_protocol.h usability... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+$ac_includes_default
+#include <zbmi_protocol.h>
+_ACEOF
+rm -f conftest.$ac_objext
+if { (ac_try="$ac_compile"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_compile") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && {
+         test -z "$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       } && test -s conftest.$ac_objext; then
+  ac_header_compiler=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+        ac_header_compiler=no
+fi
+
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_compiler" >&5
+$as_echo "$ac_header_compiler" >&6; }
+
+# Is the header present?
+{ $as_echo "$as_me:$LINENO: checking zbmi_protocol.h presence" >&5
+$as_echo_n "checking zbmi_protocol.h presence... " >&6; }
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <zbmi_protocol.h>
+_ACEOF
+if { (ac_try="$ac_cpp conftest.$ac_ext"
+case "(($ac_try" in
+  *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;;
+  *) ac_try_echo=$ac_try;;
+esac
+eval ac_try_echo="\"\$as_me:$LINENO: $ac_try_echo\""
+$as_echo "$ac_try_echo") >&5
+  (eval "$ac_cpp conftest.$ac_ext") 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  $as_echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } >/dev/null && {
+         test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" ||
+         test ! -s conftest.err
+       }; then
+  ac_header_preproc=yes
+else
+  $as_echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+  ac_header_preproc=no
+fi
+
+rm -f conftest.err conftest.$ac_ext
+{ $as_echo "$as_me:$LINENO: result: $ac_header_preproc" >&5
+$as_echo "$ac_header_preproc" >&6; }
+
+# So?  What about this header?
+case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in
+  yes:no: )
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: accepted by the compiler, rejected by the preprocessor!" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: accepted by the compiler, rejected by the preprocessor!" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: proceeding with the compiler's result" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: proceeding with the compiler's result" >&2;}
+    ac_header_preproc=yes
+    ;;
+  no:yes:* )
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: present but cannot be compiled" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: present but cannot be compiled" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h:     check for missing prerequisite headers?" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h:     check for missing prerequisite headers?" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: see the Autoconf documentation" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: see the Autoconf documentation" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h:     section \"Present But Cannot Be Compiled\"" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h:     section \"Present But Cannot Be Compiled\"" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: proceeding with the preprocessor's result" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: proceeding with the preprocessor's result" >&2;}
+    { $as_echo "$as_me:$LINENO: WARNING: zbmi_protocol.h: in the future, the compiler will take precedence" >&5
+$as_echo "$as_me: WARNING: zbmi_protocol.h: in the future, the compiler will take precedence" >&2;}
+
+    ;;
+esac
+{ $as_echo "$as_me:$LINENO: checking for zbmi_protocol.h" >&5
+$as_echo_n "checking for zbmi_protocol.h... " >&6; }
+if test "${ac_cv_header_zbmi_protocol_h+set}" = set; then
+  $as_echo_n "(cached) " >&6
+else
+  ac_cv_header_zbmi_protocol_h=$ac_header_preproc
+fi
+{ $as_echo "$as_me:$LINENO: result: $ac_cv_header_zbmi_protocol_h" >&5
+$as_echo "$ac_cv_header_zbmi_protocol_h" >&6; }
+
+fi
+if test "x$ac_cv_header_zbmi_protocol_h" = x""yes; then
+  :
+else
+  { { $as_echo "$as_me:$LINENO: error: Header zbmi_protocol.h not found." >&5
+$as_echo "$as_me: error: Header zbmi_protocol.h not found." >&2;}
+   { (exit 1); exit 1; }; }
+fi
+
+
+        CPPFLAGS="$save_cppflags"
+        BUILD_ZOID=1
+    fi
+
+
+
+
 { $as_echo "$as_me:$LINENO: checking for fgetxattr extra arguments" >&5
 $as_echo_n "checking for fgetxattr extra arguments... " >&6; }
@@ -19324 +20152 @@
 done
 
-ac_config_files="$ac_config_files include/pvfs2.h Makefile module.mk src/apps/admin/module.mk src/apps/admin/pvfs2-config src/apps/karma/module.mk src/apps/vis/module.mk src/apps/fuse/module.mk src/apps/kernel/linux/module.mk src/apps/user/module.mk src/io/trove/module.mk src/io/trove/trove-handle-mgmt/module.mk src/io/trove/trove-dbpf/module.mk src/common/misc/module.mk src/common/quickhash/module.mk src/common/quicklist/module.mk src/common/dotconf/module.mk src/common/id-generator/module.mk src/common/gossip/module.mk src/common/gen-locks/module.mk src/common/llist/module.mk src/common/statecomp/module.mk src/common/events/module.mk src/common/mgmt/module.mk src/io/bmi/module.mk src/io/bmi/bmi_tcp/module.mk src/io/bmi/bmi_gm/module.mk src/io/bmi/bmi_mx/module.mk src/io/bmi/bmi_ib/module.mk src/io/bmi/bmi_portals/module.mk src/io/description/module.mk src/io/flow/module.mk src/io/flow/flowproto-bmi-trove/module.mk src/io/flow/flowproto-template/module.mk src/io/flow/flowproto-dump-offsets/module.mk src/io/flow/flowproto-bmi-cache/module.mk src/io/buffer/module.mk src/io/job/module.mk src/io/dev/module.mk src/proto/module.mk src/server/module.mk src/server/request-scheduler/module.mk src/client/sysint/module.mk src/kernel/linux-2.6/Makefile src/kernel/linux-2.4/Makefile doc/module.mk doc/coding/module.mk doc/design/module.mk doc/random/module.mk examples/pvfs2-server.rc doc/doxygen/pvfs2-doxygen.conf"
+ac_config_files="$ac_config_files include/pvfs2.h Makefile module.mk src/apps/admin/module.mk src/apps/admin/pvfs2-config src/apps/karma/module.mk src/apps/vis/module.mk src/apps/fuse/module.mk src/apps/kernel/linux/module.mk src/apps/user/module.mk src/io/trove/module.mk src/io/trove/trove-handle-mgmt/module.mk src/io/trove/trove-dbpf/module.mk src/common/misc/module.mk src/common/quickhash/module.mk src/common/quicklist/module.mk src/common/dotconf/module.mk src/common/id-generator/module.mk src/common/gossip/module.mk src/common/gen-locks/module.mk src/common/llist/module.mk src/common/statecomp/module.mk src/common/events/module.mk src/common/mgmt/module.mk src/io/bmi/module.mk src/io/bmi/bmi_tcp/module.mk src/io/bmi/bmi_gm/module.mk src/io/bmi/bmi_mx/module.mk src/io/bmi/bmi_ib/module.mk src/io/bmi/bmi_portals/module.mk src/io/bmi/bmi_zoid/module.mk src/io/description/module.mk src/io/flow/module.mk src/io/flow/flowproto-bmi-trove/module.mk src/io/flow/flowproto-template/module.mk src/io/flow/flowproto-dump-offsets/module.mk src/io/flow/flowproto-bmi-cache/module.mk src/io/buffer/module.mk src/io/job/module.mk src/io/dev/module.mk src/proto/module.mk src/server/module.mk src/server/request-scheduler/module.mk src/client/sysint/module.mk src/kernel/linux-2.6/Makefile src/kernel/linux-2.4/Makefile doc/module.mk doc/coding/module.mk doc/design/module.mk doc/random/module.mk examples/pvfs2-server.rc doc/doxygen/pvfs2-doxygen.conf"
 
 cat >confcache <<\_ACEOF
@@ -19952 +20780 @@
     "src/io/bmi/bmi_ib/module.mk") CONFIG_FILES="$CONFIG_FILES src/io/bmi/bmi_ib/module.mk" ;;
     "src/io/bmi/bmi_portals/module.mk") CONFIG_FILES="$CONFIG_FILES src/io/bmi/bmi_portals/module.mk" ;;
+    "src/io/bmi/bmi_zoid/module.mk") CONFIG_FILES="$CONFIG_FILES src/io/bmi/bmi_zoid/module.mk" ;;
     "src/io/description/module.mk") CONFIG_FILES="$CONFIG_FILES src/io/description/module.mk" ;;
     "src/io/flow/module.mk") CONFIG_FILES="$CONFIG_FILES src/io/flow/module.mk" ;;
@@ -20764 +21593 @@
 
 if test x$BUILD_GM = x1 -o x$BUILD_MX = x1 -o x$BUILD_IB = x1 -o \
-        x$BUILD_OPENIB = x1 -o x$BUILD_PORTALS = x1 ; then
+        x$BUILD_OPENIB = x1 -o x$BUILD_PORTALS = x1 -o x$BUILD_ZOID = x1 ; then
     if test x$BUILD_BMI_TCP = x1 ; then
         { $as_echo "$as_me:$LINENO: WARNING: You have selected to build $PKGSTR to use a \"fast\" network

branches/Orange-Branch/configure.in

@@ -45 +45 @@
 AC_CONFIG_HEADER(pvfs2-config.h)
 
+AC_CHECK_HEADER([pwd.h],
+    [AC_DEFINE(HAVE_GETPWUID, 1, Define if pwd.h exists)])
 AC_CHECK_HEADER([sys/vfs.h],
                 [AC_DEFINE(HAVE_SYS_VFS_H, 1, Define if sys/vfs.h exists)])
@@ -928 +930 @@
 
 dnl
+dnl Configure bmi_zoid, if --with-zoid or a variant given.
+dnl
+AX_ZOID
+
+dnl
 dnl check if fgetxattr takes extra arguments
 AC_MSG_CHECKING([for fgetxattr extra arguments])
@@ -1129 +1136 @@
 src/io/bmi/bmi_ib/module.mk
 src/io/bmi/bmi_portals/module.mk
+src/io/bmi/bmi_zoid/module.mk
 src/io/description/module.mk
 src/io/flow/module.mk
@@ -1268 +1276 @@
 dnl
 if test x$BUILD_GM = x1 -o x$BUILD_MX = x1 -o x$BUILD_IB = x1 -o \
-        x$BUILD_OPENIB = x1 -o x$BUILD_PORTALS = x1 ; then
+        x$BUILD_OPENIB = x1 -o x$BUILD_PORTALS = x1 -o x$BUILD_ZOID = x1 ; then
     if test x$BUILD_BMI_TCP = x1 ; then
         AC_MSG_WARN([You have selected to build $PKGSTR to use a "fast" network

branches/Orange-Branch/doc/REFERENCES.bib

@@ +1 @@
+@PhdThesis{bradles-diss,
+    author = {Bradley W. Settlemyer},
+    title = {A Study of Client-based Caching for Parallel {I/O}},
+    school = {Clemson University},
+    address = {Clemson, SC},
+    year = {2009},
+    month = {August},
+}
+
+@conference{hadoop-pvfs,
+    title={In Search of an {API} for Scalable File Systems: Under the table or
+    above it?},
+    author={Swapnil Patil and Garth A. Gibson and Gregory R. Ganger and
+    Julio Lopez and Milo Polte and Wittawat Tantisiroj and Lin Xiao},
+    booktitle={USENIX HotCloud Workshop 2009},
+    month={June},
+    year={2009}
+}
+
+@conference{syscall-diag,
+    title={System-Call Based Problem Diagnosis for {PVFS}},
+    author={Michael P. Kasick and Keith A. Bare and Eugene E. Marinelli III
+    and Jiaqi Tan and Rajeev Gandhi and Priya Narasimhan},
+    booktitle={Proceedings of the 5th Workshop on Hot Topics in System
+    Dependability (HotDep '09)},
+    month={June},
+    year={2009}
+}
+
+@conference{coord-access-toappear-iasds09,
+    title={Interfaces for Coordinated Access in the File System},
+    author={Sam Lang and Robert Latham and Dries Kimpe and Robert Ross},
+    booktitle={Proceedings of 2009 Workshop on Interfaces and
+    Architectures for Scientific Data Storage},
+    month={September},
+    year={2009},
+    note={(\textbf{To Appear})}
+}
+
+@article{dyn-fs-semantics,
+    author = {Michael Kuhn and Julian Martin Kunkel and Thomas Ludwig},
+    title = {Dynamic file system semantics to enable metadata optimizations
+    in {PVFS}},
+    journal ={Concurrency and Computation: Practice and Experience},
+    year = {2009},
+    publisher = {John Wiley and Sons, Ltd},
+}
+
+@conference{pvfs-bgp-toappear-sc09,
+    title={{I/O} Performance Challenges at Leadership Scale},
+    author={Samuel Lang and Philip Carns and Robert Latham and Robert Ross and Kevin Harms and William Allcock},
+    booktitle={Proceedings of Supercomputing},
+    month={November},
+    year={2009},
+    note={(\textbf{To Appear})}
+}
+
+@article{IPDPS.2009.5161070,
+    author = {Xuechen Zhang and Song Jiang and Kei Davis},
+    title = {Making resonance a common case: A high-performance implementation
+    of collective I/O on parallel file systems},
+    journal ={Parallel and Distributed Processing Symposium, International},
+    volume = {0},
+    year = {2009},
+    pages = {1-12},
+    doi = {http://doi.ieeecomputersociety.org/10.1109/IPDPS.2009.5161070},
+    publisher = {IEEE Computer Society},
+    address = {Los Alamitos, CA, USA},
+}
 
 @inproceedings{SmallFilesIPDPS09,
@@ -10 +79 @@
 @inproceedings{PosixExtTReport,
  author = {M. Vilayannur and  S. Lang and  R. Ross and  R. Klundt  and L. Ward},
- title = {Extending the POSIX I/O Interface: A Parallel File System Perspective},
+ title = {Extending the {POSIX I/O} Interface: A Parallel File System Perspective},
  booktitle = {Technical Memorandum ANL/MCS-TM-302},
  year = {2008},
@@ -37 +106 @@
 @inproceedings{OSDCluster08,
  author = {N. Ali and  A. Devulapalli and  D. Dalessandro and  P. Wyckoff and  P. Sadayappan},
- title = {An OSD-based Approach to Managing Directory Operations in Parallel File Systems},
+ title = {An {OSD-based} Approach to Managing Directory Operations in Parallel File Systems},
  booktitle = {IEEE International Conference on Cluster Computing},
  year = {2008},
@@ -73 +142 @@
 }
 
+@inproceedings{kunkel:bottlenecks,
+  author={Julian Kunkel and Thomas Ludwig},
+  title={Bottleneck Detection in Parallel File Systems with Trace-Based Performance Monitoring},
+  booktitle={Euro-Par 2008: Proceedings of the 14th International Euro-Par Conference on Parallel Processing},
+  year = {2008},
+  pages={212--221}
+}
+
+@inproceedings{ludwig:pioviz,
+  author={Thomas Ludwig and Stephan Krempel and Michael Kuhn and Julian Kunkel and Christian Lohse},
+  title={Analysis of the {MPI-IO} Optimization Levels with the {PIOViz} Jumpshot Enhancement},
+  booktitle={Proceedings of EuroPVM/MPI 2007},
+  year={2007},
+  month={September},
+  pages={213--222}
+}
+
 @article{BradThesis,
  author = {Bradley W. Settlemyer},
@@ -91 +177 @@
   pages     = {26-29},
   ee        = {http://doi.acm.org/10.1145/1374596.1374604},
-  crossref  = {DBLP:conf/sc/2007pdsw},
   bibsource = {DBLP, http://dblp.uni-trier.de},
   url = {http://www.pdl.cmu.edu/PDL-FTP/HECStorage/sc07-patil.pdf}
@@ -621 +706 @@
    booktitle = {HPCASIA '04: Proceedings of the High Performance Computing and Grid in Asia Pacific Region, Seventh International Conference},
    year = {2004},
-   isbn = {0-7695-2138-X},
    pages = {136--139},
    publisher = {IEEE Computer Society},

branches/Orange-Branch/doc/add-server-req

@@ -38 +38 @@
                         PINT_server_op  -- only if required
 
-5) write state machine -- se details below
+5) write state machine -- see details below
                 src/server/<reqname>.sm
                         each state machine added must have a delcaration in
                                 src/server/pvfs2-server.h
 
-6) update request scheduler
-                src/server/request-scheduler/request-scheduler.c
-                        PINT_req_sched_target_handle()
-
-7) add entry to server operation parameters table
-                src/server/pvfs2-server.c
-                        init_req_table -- see src/server/prelude.sm
-                        prelude_perm_check()
-
-8) add entry in final response state machine
-                src/server/final-response.sm
-                        s_req_resp_type_map
-
+6) update src/server/pvfs2-server-req.c
+                add new entry to PINT_server_req_table[]
+                reference the params structure from the state machine
 
 

branches/Orange-Branch/doc/pvfs2-faq.tex

@@ -319 +319 @@
 \url{http://www.pvfs.org/}.  Documentation (including this FAQ) is also
 available in the \texttt{doc} subdirectory of the PVFS source distribution.
+Please reference \texttt{pvfs2-logging.txt} to understand more about PVFS'
+informational messages, where the logs exist, and how to turn logging
+on and off.
 
 \subsection{What should I do if I have a problem?}
@@ -713 +716 @@
 pre-release kernel, but do make an effort to publish necessary patches once a
 kernel is officially released. 
+
+\subsection{Does PVFS work with Open-MX?}
+\label{sec:open-mx}
+
+Yes, PVFS does work with Open-MX.  To use Open-MX, configure PVFS with
+the the same arguments that you would use for a normal MX installation:
+``--disable-bmi-tcp'' and ``--with-mx=PATH''.  In addition, however, you
+must set the ``MX\_IMM\_ACK'' environment variable to ``1'' before starting
+the pvfs2-server or pvfs2-client daemons.  This is necessary in order to
+account for differences in how MX and Open-MX handle message progression by
+default.
 
 %

branches/Orange-Branch/doc/pvfs2-logging.txt

@@ +1 @@
+PVFS Logging
+============
+
+This document describes log files produced by PVFS and how to control
+what messages are included in them.
+
+PVFS Log Format
+---------------
+
+The log messages from all PVFS components except for the kernel module
+are in the following format: 
+
+    [<type> <timestamp>] LOG MESSAGE
+
+The type will be one of 4 different letters depending on what type of
+log message it is:
+
+    D - DEBUG 
+    E - ERROR 
+    A - ACCESS LOGGING
+    P - PERFORMANCE COUNTER 
+
+The timestamp defaults to showing the date, hour, and minute that the
+log message was generated.  The timestamp format can be modified to one
+of the following styles by using the --logstamp argument to pvfs2-client
+or the LogStamp field in the pvfs2-server config file:
+
+    datetime: (default, as described above)
+    usec: shows time with microsecond granularity (but no date)
+    none: no time stamp
+    thread: includes thread ID with each message
+
+PVFS Log Locations
+------------------
+
+The pvfs2-server daemon writes log messages to /tmp/pvfs2-server.log by
+default.  A different output file can be specified using the LogFile
+parameter in the configuration file.  The logs can also be sent to syslog
+instead by adding "LogType syslog" to the configuration file.
+
+The pvfs2-client daemon writes log messages to /tmp/pvfs2-client.log by
+default.  This can be overridden using the --logfile or --logtype command
+line arguments to pvfs2-client.
+
+The PVFS kernel module (pvfs2.ko) generates log messages to dmesg and/or
+/var/log/messages and/or /var/log/kern depending on your Linux distribution.
+
+The PVFS client library (libpvfs2) and command line utilities generate log
+messages to stderr if enabled.
+
+Logging Levels
+--------------
+
+All PVFS components log critical error messages automatically.  However, 
+you can also turn on additional logging for debugging purposes.  This is
+controlled by specifying which logging "masks" should be enabled.
+
+You can see a list of available pvfs2-server, client library, and
+pvfs2-client logging levels by running the pvfs2-set-debugmask utility
+with no arguments.
+
+You can see a list of available kernel module logging masks and client
+logging masks by running "cat /proc/sys/pvfs2/debug-help".
+
+The "verbose" mask is commonly used to turn on most of the logging
+levels that are useful for debugging problems.
+
+Changing the log mask for pvfs2-server
+--------------------------------------
+
+Use the EventLogging field in the configuration file to specify
+a comma separated list of masks to enable.  You can also use the
+pvfs2-set-debugmask command line utility to change the mask dynamically
+without restarting the server.
+
+Changing the log mask for libpvfs and command line utilities
+------------------------------------------------------------
+
+Set the PVFS2_DEBUGMASK environment variable to a comma separated list of
+client-appropriate masks prior to launching the application.
+
+
+Changing the log mask for the kernel module
+-------------------------------------------
+
+There are three ways to set the debugging level for the kernel module:
+
+1.  Set module_parm_debug_mask parameter when the kernel module is
+loaded.  
+
+2.  Set the environment variable PVFS2_KMODMASK before starting the 
+pvfs2-client.  NOTE:  the kernel module must be loaded before starting
+the client-core.
+
+3.  Write a debug string to /proc/sys/pvfs2/kernel-debug after the kernel
+module is loaded.
+
+
+Options 1 and 2 allow the kernel debug mask to be set ONLY when PVFS is started,
+while option 3 allows the kernel debug mask to be modified while PVFS is 
+running.  Thus, option 3 dynamically updates the kernel debug mask, 
+immediately turning on the debugging options specified, and REPLACES the
+existing debug mask.  Whenever you modify the kernel debug mask using
+option 3, an informational message is printed to the system log file, 
+giving both its numerical value and a comma-separated list of keywords representing
+the areas of debugging just turned on.
+
+Options 1 and 2 require the user to specify a numerical value that is
+an OR'd list of gossip debug values.  These values can be found in
+include/pvfs2-debug.h.  For example, to load the kernel module with
+"file" debugging turned on, issue the following command:
+
+insmod pvfs2.ko module_parm_debug_mask=4
+
+The 4 is the value of GOSSIP_FILE_DEBUG, and module_parm_debug_mask is the kernel
+module's input parameter for the kernel debug mask.  To turn on multiple areas,for
+example, file and dcache, set module_parm_debug_mask = (GOSSIP_FILE_DEBUG | GOSSIP_DCACHE_DEBUG) =
+(4 | 128) = 132.  Its string equivalent would be "file,dcache".
+
+An informational message is displayed in the system log whenever you load the kernel 
+module, giving you the kernel debug mask's numerical value and its string 
+equivalent.  Be aware that you can modify this value later using option 3.
+
+To set the kernel debug mask using PVFS2_KMODMASK, create a global environment 
+variable and set it equal to the desired numerical value.  When the pvfs2-client
+is started, the kernel debug mask and its string equivalent will be modified.  Note that 
+PVFS2_KMODMASK will override any value set by the kernel module load process.  Again,
+option 3 allows you to change the debug mask at any time.
+
+To set the kernel debug mask using the /proc variable, write a debug string to 
+/proc/sys/pvfs2/kernel-debug.  Example:  echo "file,dcache" > /proc/sys/pvfs2/kernel-debug.
+An informational message will be written to the system log file displaying the new
+kernel debug mask and its string equivalent.
+
+To see the kernel debug mask without looking in the system log, issue a "cat" on
+/proc/sys/pvfs2/kernel-debug and you will see the string equivalent of the kernel
+debug mask. 
+
+A helper /proc variable, /proc/sys/pvfs2/debug-help, will display a list of valid
+keywords for both the kernel and client debug masks, when you issue a "cat" on it.  These 
+keywords are used to build the string that represent the areas of debugging that you 
+want turned on.
+
+
+
+Changing the log mask for the client module
+-------------------------------------------
+
+There are three ways to set the debugging level for the pvfs2-client:
+
+1.  Set --gossip-mask=MASK_LIST on the command line when starting the client.  This
+list can be overridden by PVFS2_DEBUGMASK or by setting the /proc variable client-debug. 
+
+2.  Write a debug string to /proc/sys/pvfs2/client-debug after starting the client. This
+will override any value set by --gossip-mask on the command line and any value set by
+PVFS2_DEBUGMASK.
+
+3.  Set the environment variable PVFS2_DEBUGMASK before starting the client.  This will
+override any value set by --gossip-mask on the command line.
+
+
+Options 1 and 2 require a string of comma-separated keywords to set the client debug mask.
+For example:
+
+./pvfs2-client --gossip-mask="server,trove" -p ./pvfs2-client-core
+NOTE: after kernel module is loaded and during client startup.
+
+echo "server,trove" > /proc/sys/pvfs2/client-debug
+NOTE: after kernel module is loaded and client is started.
+
+A list of client debug keywords can be found in include/pvfs2-debug.h or by accessing
+the /proc/sys/pvfs2/debug-help variable after the kernel module is loaded. Example:
+
+cat /proc/sys/pvfs2/debug-help
+
+
+When the client starts, the client debug mask information is sent to the kernel module
+where a local version of the mask and its string equivalent is maintained.  This process
+writes an informational message to the system log file displaying the numerical value of
+the client debug mask and its string equivalent.  You can also see the mask's current string
+equivalent by issuing the following:
+
+cat /proc/sys/pvfs2/client-debug.
+
+
+Whenever you modify the client debug mask after the client has started, an informational message will
+be written to the system log file displaying the new numerical value and string equivalent.  At any
+time, once PVFS is running, you can view the client debug mask using the "cat" statement above without
+having to look in the system log file for the last modification.
+
+

branches/Orange-Branch/doc/pvfs2-quickstart.tex

@@ -939 +939 @@
 and eventually to the kernel logs.
 One could also set the kmod diagnostic level when the kernel module is loaded
-like so, insmod pvfs2.ko gossip\_debug\_mask=<diagnostic level>.
+like so, insmod pvfs2.ko module\_parm\_debug\_mask=<diagnostic level>.
 The diagnostic level will be a bitwise OR of values specified in pvfs2-debug.h.
+For more information on setting the kernel or client debug mask, see
+\texttt{doc/pvfs2-logging.txt} in the PVFS source tree.
 
 \section{ROMIO Support}

branches/Orange-Branch/examples/heartbeat/hardware-specific/Filesystem-qla-monitor

@@ -644 +644 @@
       # shoot ourselves in the head
       return $OCF_ERR_GENERIC
-    else
-      return $ORIG_RET
-    fi    
+    fi
+    dd bs=4096 count=1 if=/dev/zero oflag=direct of=${MOUNTPOINT}/filesystem.monitor
+    if [ $? -ne 0 ]; then
+      ocf_log err "Error: Unable to write to $DEVICE on node `hostname`."
+      /usr/bin/logger -p user.err -t PVFS2 "Error: Unable to write to $DEVICE on node `hostname`, failing over."
+      /usr/bin/fs-instance-alarm.pl --fs-name $fsname --ce `hostname` --type PVFS2_HA --msg "Error: Unable to write to $DEVICE on node `hostname`, failing over."
+      # shoot ourselves in the head
+      /usr/bin/fs-power-control.pl --fs-name $fsname --conf-dir $conf_dir --host `hostname` --command reboot
+      return $OCF_ERR_GENERIC
+    fi
   fi
   return $ORIG_RET
@@ -718 +725 @@
 fi
 
+OP=$1
+
+# These operations do not require instance parameters
+case $OP in
+  meta-data)            meta_data
+                        exit $OCF_SUCCESS
+                        ;;
+  usage)                usage
+                        exit $OCF_SUCCESS
+                        ;;
+esac
+
 if 
   [ -z "$OCF_RESKEY_fsname" ]
@@ -742 +761 @@
 fsname=$OCF_RESKEY_fsname
 conf_dir=$OCF_RESKEY_conf_dir
-
-OP=$1
-
-# These operations do not require instance parameters
-case $OP in
-  meta-data)            meta_data
-                        exit $OCF_SUCCESS
-                        ;;
-  usage)                usage
-                        exit $OCF_SUCCESS
-                        ;;
-esac
 
 blockdevice=no

branches/Orange-Branch/include/pvfs2-debug.h

@@ -78 +78 @@
 #define GOSSIP_MIRROR_DEBUG            ((uint64_t)1 << 56)
 
-/* NOTE: if you want your gossip flag to be controllable from 
- * pvfs2-set-debugmask you have to add it in
- * src/common/misc/pvfs2-debug.c
- */
-
 #define GOSSIP_BMI_DEBUG_ALL (uint64_t)                               \
 (GOSSIP_BMI_DEBUG_TCP + GOSSIP_BMI_DEBUG_CONTROL +                    \
  GOSSIP_BMI_DEBUG_GM + GOSSIP_BMI_DEBUG_OFFSETS + GOSSIP_BMI_DEBUG_IB \
  + GOSSIP_BMI_DEBUG_MX + GOSSIP_BMI_DEBUG_PORTALS)
-
-uint64_t PVFS_debug_eventlog_to_mask(
-    const char *event_logging);
 
 const char *PVFS_debug_get_next_debug_keyword(

branches/Orange-Branch/include/pvfs2-encode-stubs.h

@@ -39 +39 @@
  */
 #ifdef __PINT_REQPROTO_ENCODE_FUNCS_C
-#  include "src/proto/endecode-funcs.h"
+#include "src/proto/endecode-funcs.h"
 #else  /* __PINT_REQPROTO_ENCODE_FUNCS_C */
 
 /* dummy declarations to turn off functions */
-#define endecode_fields_1(n,t1,x1)
-#define endecode_fields_1_struct(n,t1,x1)
-#define endecode_fields_2(n,t1,x1,t2,x2)
-#define endecode_fields_2_struct(n,t1,x1,t2,x2)
-#define endecode_fields_3(n,t1,x1,t2,x2,t3,x3)
-#define endecode_fields_3_struct(n,t1,x1,t2,x2,t3,x3)
-#define endecode_fields_4(n,t1,x1,t2,x2,t3,x3,t4,x4)
-#define endecode_fields_4_struct(n,t1,x1,t2,x2,t3,x3,t4,x4)
-#define endecode_fields_5(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5)
-#define endecode_fields_5_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5)
-#define endecode_fields_6(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6)
-#define endecode_fields_7_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7)
-#define endecode_fields_8_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8)
-#define endecode_fields_9_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9)
-#define endecode_fields_10_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10)
-#define endecode_fields_11_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10,t11,x11)
-#define endecode_fields_12(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10,t11,x11,t12,x12)
+#define endecode_fields_1(n,t1,x1) struct endecode_fake_struct
+#define endecode_fields_1_struct(n,t1,x1) struct endecode_fake_struct
+#define endecode_fields_2(n,t1,x1,t2,x2) struct endecode_fake_struct
+#define endecode_fields_2_struct(n,t1,x1,t2,x2) struct endecode_fake_struct
+#define endecode_fields_3(n,t1,x1,t2,x2,t3,x3) struct endecode_fake_struct
+#define endecode_fields_3_struct(n,t1,x1,t2,x2,t3,x3) struct endecode_fake_struct
+#define endecode_fields_4(n,t1,x1,t2,x2,t3,x3,t4,x4) struct endecode_fake_struct
+#define endecode_fields_4_struct(n,t1,x1,t2,x2,t3,x3,t4,x4) struct endecode_fake_struct
+#define endecode_fields_5(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5) struct endecode_fake_struct
+#define endecode_fields_5_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5) struct endecode_fake_struct
+#define endecode_fields_6(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6) struct endecode_fake_struct
+#define endecode_fields_7_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7) struct endecode_fake_struct
+#define endecode_fields_8_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8) struct endecode_fake_struct
+#define endecode_fields_9_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9) struct endecode_fake_struct
+#define endecode_fields_10_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10) struct endecode_fake_struct
+#define endecode_fields_11_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10,t11,x11) struct endecode_fake_struct
+#define endecode_fields_12(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,t6,x6,t7,x7,t8,x8,t9,x9,t10,x10,t11,x11,t12,x12) struct endecode_fake_struct
 
-#define endecode_fields_1a(n,t1,x1,tn1,n1,ta1,a1)
-#define endecode_fields_1a_struct(n,t1,x1,tn1,n1,ta1,a1)
-#define endecode_fields_1aa_struct(n,t1,x1,tn1,n1,ta1,a1,ta2,a2)
-#define endecode_fields_2a_struct(n,t1,x1,t2,x2,tn1,n1,ta1,a1)
-#define endecode_fields_2aa_struct(n,t1,x1,t2,x2,tn1,n1,ta1,a1,ta2,a2)
-#define endecode_fields_3a_struct(n,t1,x1,t2,x2,t3,x3,tn1,n1,ta1,a1)
-#define endecode_fields_4aa_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,tn1,n1,ta1,a1,ta2,a2)
-#define endecode_fields_1a_1a_struct(n,t1,x1,tn1,n1,ta1,a1,t2,x2,tn2,n2,ta2,a2)
-#define endecode_fields_4a_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,tn1,n1,ta1,a1)
-#define endecode_fields_5a_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,tn1,n1,ta1,a1)
+#define endecode_fields_1a(n,t1,x1,tn1,n1,ta1,a1) struct endecode_fake_struct
+#define endecode_fields_1a_struct(n,t1,x1,tn1,n1,ta1,a1) struct endecode_fake_struct
+#define endecode_fields_1aa_struct(n,t1,x1,tn1,n1,ta1,a1,ta2,a2) struct endecode_fake_struct
+#define endecode_fields_2a_struct(n,t1,x1,t2,x2,tn1,n1,ta1,a1) struct endecode_fake_struct
+#define endecode_fields_2aa_struct(n,t1,x1,t2,x2,tn1,n1,ta1,a1,ta2,a2) struct endecode_fake_struct
+#define endecode_fields_3a_struct(n,t1,x1,t2,x2,t3,x3,tn1,n1,ta1,a1) struct endecode_fake_struct
+#define endecode_fields_4aa_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,tn1,n1,ta1,a1,ta2,a2) struct endecode_fake_struct
+#define endecode_fields_1a_1a_struct(n,t1,x1,tn1,n1,ta1,a1,t2,x2,tn2,n2,ta2,a2) struct endecode_fake_struct
+#define endecode_fields_4a_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,tn1,n1,ta1,a1) struct endecode_fake_struct
+#define endecode_fields_5a_struct(n,t1,x1,t2,x2,t3,x3,t4,x4,t5,x5,tn1,n1,ta1,a1) struct endecode_fake_struct
 
-#define encode_enum_union_2_struct(name, ename, uname, ut1, un1, en1, ut2, un2, en2)
+#define encode_enum_union_2_struct(name, ename, uname, ut1, un1, en1, ut2, un2, en2) struct endecode_fake_struct
 
 #endif  /* __PINT_REQPROTO_ENCODE_FUNCS_C */

branches/Orange-Branch/include/pvfs2-mgmt.h

@@ -75 +75 @@
     int32_t, keyval_queue,
     int32_t, reqsched,
-    skip4,)
+    skip4,);
 
 /* low level information about individual server level objects */
@@ -94 +94 @@
   skip4,,
   PVFS_size, b_size,
-  PVFS_handle, dirdata_handle)
+  PVFS_handle, dirdata_handle);
 
 /* individual datapoint from event monitoring */
@@ -116 +116 @@
     int32_t, tv_sec,
     int32_t, tv_usec,
-    skip4,)
+    skip4,);
 
 /* values which may be or'd together in the flags field above */

branches/Orange-Branch/include/pvfs2-types.h

@@ -103 +103 @@
     ENCODING_DIRECT = 1,
     ENCODING_LE_BFIELD = 2,
-    ENCODING_XDR = 3,
+    ENCODING_XDR = 3
 };
+
 /* these values must correspond to the defined encoding types above */
 #define ENCODING_INVALID_MIN                    0
@@ -502 +503 @@
     } u;
 };
+
 encode_enum_union_2_struct(
     PVFS_mgmt_setparam_value,
     type, u,
     uint64_t, value,        PVFS_MGMT_PARAM_TYPE_UINT64,
-    string,   string_value, PVFS_MGMT_PARAM_TYPE_STRING)
+    string,   string_value, PVFS_MGMT_PARAM_TYPE_STRING);
 
 enum PVFS_server_mode

branches/Orange-Branch/maint/config/kernel.m4

@@ -544 +544 @@
         fi
 
-        dnl Test to see if sysctl proc handlers have a 6th argument
-        AC_MSG_CHECKING(for 6th argument to sysctl proc handlers)
-        dnl if this test passes, there is a 6th argument
+        dnl the proc handler functions have changed over the years.
+        dnl pre-2.6.8: proc_handler(ctl_table       *ctl,
+        dnl                         int             write,
+        dnl                         struct file     *filp,
+        dnl                         void            *buffer,
+        dnl                         size_t          *lenp)
+        dnl
+        dnl 2.6.8-2.6.31: proc_handler(ctl_table       *ctl,
+        dnl                            int             write,
+        dnl                            struct file     *filp,
+        dnl                            void            *buffer,
+        dnl                            size_t          *lenp,
+        dnl                            loff_t          *ppos)
+        dnl > 2.6.31: proc_handler(ctl_table       *ctl,
+        dnl                        int             write,
+        dnl                        void            *buffer,
+        dnl                        size_t          *lenp,
+        dnl                        loff_t          *ppos)
+ 
+        dnl Test to see if sysctl proc handlers have a file argument
+        AC_MSG_CHECKING(for file argument to sysctl proc handlers)
         AC_TRY_COMPILE([
             #define __KERNEL__
@@ -552 +570 @@
             #include <linux/sysctl.h>
             ], [
-            proc_dointvec_minmax(NULL, 0, NULL, NULL, NULL, NULL);
+                struct ctl_table * ctl = NULL;
+                int write = 0;
+                struct file * filp = NULL;
+                void __user * buffer = NULL;
+                size_t * lenp = NULL;
+                loff_t * ppos = NULL;
+
+                proc_dointvec_minmax(ctl, write, filp, buffer, lenp, ppos);
             ],
             AC_MSG_RESULT(yes)
-            AC_DEFINE(HAVE_PROC_HANDLER_SIX_ARG, 1, Define if sysctl proc handlers have 6th argument),
+            AC_DEFINE(HAVE_PROC_HANDLER_FILE_ARG, 1, Define if sysctl proc handlers have 6th argument),
+            AC_MSG_RESULT(no)
+            )
+
+        AC_MSG_CHECKING(for ppos argument to sysctl proc handlers)
+        dnl if this test passes, there is a ppos argument
+        AC_TRY_COMPILE([
+            #define __KERNEL__
+            #include <linux/fs.h>
+            #include <linux/sysctl.h>
+            ], [
+                struct ctl_table * ctl = NULL;
+                int write = 0;
+                void __user * buffer = NULL;
+                size_t * lenp = NULL;
+                loff_t * ppos = NULL;
+
+                proc_dointvec_minmax(ctl, write, buffer, lenp, ppos);
+            ],
+            AC_MSG_RESULT(yes)
+            AC_DEFINE(HAVE_PROC_HANDLER_PPOS_ARG, 1, Define if sysctl proc handlers have ppos argument),
             AC_MSG_RESULT(no)
             )
@@ -1174 +1219 @@
         )
 
+        AC_MSG_CHECKING(for s_dirty in struct super_block)
+        AC_TRY_COMPILE([
+                #define __KERNEL__
+                #include <linux/fs.h>
+        ], [
+                struct super_block *s;
+                list_empty(&s->s_dirty);
+        ],
+        AC_MSG_RESULT(yes)
+        AC_DEFINE(HAVE_SB_DIRTY_LIST, 1, [Define if struct super_block has s_dirty list]),
+        AC_MSG_RESULT(no)
+        )
+
         dnl newer 2.6 kernels (2.6.29-ish) use current_fsuid() macro instead
         dnl of accessing task struct fields directly
@@ -1192 +1250 @@
         CFLAGS=$tmp_cflags
 
+        dnl 2.6.32 added a mandatory name field to the bdi structure
+        AC_MSG_CHECKING(if kernel backing_dev_info struct has a name field)
+        AC_TRY_COMPILE([
+                #define __KERNEL__
+                #include <linux/fs.h>
+                #include <linux/backing-dev.h>
+        ], [
+                struct backing_dev_info foo = 
+                {
+                    .name = "foo"
+                };
+        ],
+        AC_MSG_RESULT(yes)
+        AC_DEFINE(HAVE_BACKING_DEV_INFO_NAME, 1, [Define if kernel backing_dev_info struct has a name field]),
+        AC_MSG_RESULT(no)
+        )
+
+        dnl some 2.6 kernels have functions to explicitly initialize bdi structs
+        tmp_cflags=$CFLAGS
+        CFLAGS="$CFLAGS -Werror"
+        AC_MSG_CHECKING(for bdi_init)
+        AC_TRY_COMPILE([
+                #define __KERNEL__
+                #include <linux/fs.h>
+                #include <linux/backing-dev.h>
+        ], [
+                int ret = bdi_init(NULL);
+        ],
+        AC_MSG_RESULT(yes)
+        AC_DEFINE(HAVE_BDI_INIT, 1, [Define if bdi_init function is present]),
+        AC_MSG_RESULT(no)
+        )
+        CFLAGS=$tmp_cflags
+
 
         CFLAGS=$oldcflags

branches/Orange-Branch/pvfs2-config.h.in

@@ -33 +33 @@
 /* Define to 1 if you have the <attr/xattr.h> header file. */
 #undef HAVE_ATTR_XATTR_H
+
+/* Define if kernel backing_dev_info struct has a name field */
+#undef HAVE_BACKING_DEV_INFO_NAME
+
+/* Define if bdi_init function is present */
+#undef HAVE_BDI_INIT
 
 /* Define if struct backing_dev_info in kernel has memory_backed */
@@ -126 +132 @@
 #undef HAVE_GETMNTENT
 
+/* Define if pwd.h exists */
+#undef HAVE_GETPWUID
+
 /* Define if struct super_operations in kernel has get_fs_key callback */
 #undef HAVE_GET_FS_KEY_SUPER_OPERATIONS
@@ -265 +274 @@
 
 /* Define if sysctl proc handlers have 6th argument */
-#undef HAVE_PROC_HANDLER_SIX_ARG
+#undef HAVE_PROC_HANDLER_FILE_ARG
+
+/* Define if sysctl proc handlers have ppos argument */
+#undef HAVE_PROC_HANDLER_PPOS_ARG
 
 /* Define if have PtlACEntry with jid argument. */
@@ -304 +316 @@
    tree_lock */
 #undef HAVE_RW_LOCK_TREE_ADDR_SPACE_STRUCT
+
+/* Define if struct super_block has s_dirty list */
+#undef HAVE_SB_DIRTY_LIST
 
 /* Define to 1 if you have the <SDL/SDL_ttf.h> header file. */

branches/Orange-Branch/src/apps/admin/pvfs2-ls.c

@@ -34 +34 @@
 
 /*
+  Define the maximum length of a single line of output. This is about the 
+  size of 256 maximum path segments, a file name, and attributes.
+ */
+#define ENTRY_MAX          66560
+
+/*
   arbitrarily restrict the number of paths
   that this ls version can take as arguments
 */
 #define MAX_NUM_PATHS       8
+
+/*
+  Max length of the fully formatted date/time fields
+*/
+#define MAX_TIME_LENGTH    128
+
+/*
+  Length of the formatted date/time for --all-times option
+*/
+#define ALL_TIMES_LENGTH    25
 
 /* optional parameters, filled in by parse_args() */
@@ -53 +69 @@
     int list_no_owner;
     int list_inode;
+    int list_all_times;
     int list_use_si_units;
     char *start[MAX_NUM_PATHS];
@@ -78 +95 @@
     PVFS_sys_attr *attr,
     int attr_error,
-    struct options *opts);
+    struct options *opts,
+    char* entry_buffer);
 
 static int do_list(
@@ -84 +102 @@
     char *start,
     int fs_id,
-    struct options *opts);
+    struct options *opts,
+    char *entry_buffer);
 
 static void print_entry_attr(
@@ -90 +109 @@
     char *entry_name,
     PVFS_sys_attr *attr,
-    struct options *opts);
+    struct options *opts,
+    char *entry_buffer);
 
 #define print_dot_and_dot_dot_info_if_required(refn)        \
@@ -108 +128 @@
         else if (opts->list_long) {                         \
             print_entry(".", refn.handle,                   \
-                        refn.fs_id, NULL, 0, opts);         \
+                        refn.fs_id, NULL, 0, opts,          \
+                        entry_buffer);                      \
             print_entry(".. (faked)", refn.handle,          \
-                        refn.fs_id, NULL, 0, opts);         \
+                        refn.fs_id, NULL, 0, opts,          \
+                        entry_buffer);                      \
         }                                                   \
         else {                                              \
@@ -190 +212 @@
     char *entry_name,
     PVFS_sys_attr *attr,
-    struct options *opts)
+    struct options *opts,
+    char *entry_buffer)
 {
-    char buf[128] = {0}, *formatted_size = NULL;
-    char *formatted_owner = NULL, *formatted_group = NULL;
+    char *formatted_size = NULL;
+    char *formatted_owner = NULL, *formatted_group = NULL, *formatted_time = NULL;
     struct group *grp = NULL;
     struct passwd *pwd = NULL;
@@ -199 +222 @@
     char *owner = empty_str, *group = empty_str;
     char *inode = empty_str;
-    time_t mtime;
-    struct tm *time;    
+    time_t mtime, atime, ctime;
+    struct tm *time;
     PVFS_size size = 0;
-    char scratch_owner[16] = {0}, scratch_group[16] = {0};
+    char scratch_owner[16] = {0}, scratch_group[16] = {0}, scratch_time[MAX_TIME_LENGTH] = {0}, scratch_big_time[MAX_TIME_LENGTH] = {0};
     char scratch_size[16] = {0}, scratch_inode[16] = {0};
     char f_type = '-';
     char group_x_char = '-';
+    int num_bytes = 0;
 
     if (!opts->list_all && (entry_name[0] == '.'))
@@ -215 +239 @@
         return;
     }
+
     mtime = (time_t)attr->mtime;
     time = localtime(&mtime);
+    if(opts->list_all_times)
+    {
+        atime = (time_t)attr->atime;
+        ctime = (time_t)attr->ctime;
+
+        num_bytes = strftime( scratch_time,ALL_TIMES_LENGTH+1,"%F %H:%M:%S %z",time );
+        strncpy(scratch_big_time,scratch_time,num_bytes);
+
+        time = localtime(&atime);
+        num_bytes = strftime( scratch_time,ALL_TIMES_LENGTH+3,"  %F %H:%M:%S %z",time );
+        strncat(scratch_big_time,scratch_time,num_bytes);
+
+        time = localtime(&ctime);
+        num_bytes = strftime( scratch_time,ALL_TIMES_LENGTH+3,"  %F %H:%M:%S %z",time );
+        strncat(scratch_big_time,scratch_time,num_bytes);
+
+        format_size_string(scratch_big_time,strlen(scratch_big_time),&formatted_time,0,1);
+    }
+    else
+    {
+        strftime( scratch_time,17,"%F %H:%M",time );
+        format_size_string(scratch_time,16,&formatted_time,0,1);
+    }
 
     snprintf(scratch_owner,16,"%d",(int)attr->owner);
@@ -303 +351 @@
     }
 
-    snprintf(buf,128,"%s%c%c%c%c%c%c%c%c%c%c    1 %s %s %s "
-             "%.4d-%.2d-%.2d %.2d:%.2d %s",
+    snprintf(entry_buffer,ENTRY_MAX,"%s%c%c%c%c%c%c%c%c%c%c    1 %s %s %s "
+             "%s %s",
              inode,
              f_type,
@@ -319 +367 @@
              formatted_group,
              formatted_size,
-             (time->tm_year + 1900),
-             (time->tm_mon + 1),
-             time->tm_mday,
-             (time->tm_hour),
-             (time->tm_min),
+             formatted_time,
              entry_name);
 
@@ -338 +382 @@
         free(formatted_group);
     }
+    if (formatted_time)
+    {
+        free(formatted_time);
+    }
 
     if (attr->objtype == PVFS_TYPE_SYMLINK)
@@ -345 +393 @@
         if (opts->list_long)
         {
-            printf("%s -> %s\n", buf, attr->link_target);
+            printf("%s -> %s\n", entry_buffer, attr->link_target);
         }
         else
         {
-            printf("%s\n",buf);
+            printf("%s\n",entry_buffer);
         }
     }
     else
     {
-        printf("%s\n",buf);
+        printf("%s\n",entry_buffer);
     }
 }
@@ -364 +412 @@
     PVFS_sys_attr *attr,
     int attr_error,
-    struct options *opts)
+    struct options *opts,
+    char *entry_buffer)
 {
     int ret = -1;
@@ -406 +455 @@
                 return;
             }
-            print_entry_attr(handle, entry_name,  &getattr_response.attr, opts);
+            print_entry_attr(handle, entry_name,  &getattr_response.attr, opts, entry_buffer);
         }
         else
         {
-            print_entry_attr(handle, entry_name, attr, opts);
+            print_entry_attr(handle, entry_name, attr, opts, entry_buffer);
         }
     }
@@ -426 +475 @@
     char *start,
     int fs_id,
-    struct options *opts)
+    struct options *opts,
+    char *entry_buffer)
 {
     int i = 0, printed_dot_info = 0;
@@ -495 +545 @@
             {
                 print_entry_attr(ref.handle, segment,
-                                 &getattr_response.attr, opts);
+                                 &getattr_response.attr, opts, entry_buffer);
             }
             else
@@ -502 +552 @@
                         NULL,
                         0,
-                        opts);
+                        opts, entry_buffer);
             }
             return 0;
@@ -559 +609 @@
                     &rdplus_response.attr_array[i],
                     rdplus_response.stat_err_array[i],
-                    opts);
+                    opts, entry_buffer);
 
             PVFS_sys_attr *attr = &rdplus_response.attr_array[i];
@@ -640 +690 @@
         {
             printf("\n");
-            do_list(full_path,current->path,fs_id,opts);
+            do_list(full_path,current->path,fs_id,opts,entry_buffer);
             current = current->next;
             free(head->path);
@@ -676 +726 @@
         {"inode",0,0,0},
         {"size",0,0,0},
+        {"all-times",0,0,0},
         {0,0,0,0}
     };
@@ -744 +795 @@
                 {
                     goto list_inode;
+                }
+                else if (strcmp("all-times", cur_option) == 0)
+                {
+                    goto list_all_times;
                 }
                 else
@@ -796 +851 @@
                 tmp_opts->list_inode = 1;
                 break;
+          list_all_times:
+                tmp_opts->list_all_times = 1;
+                break;
             case 't':
                 do_timing = 1;
@@ -845 +903 @@
     fprintf(stderr,"  -n, --numeric-uid-gid      like -l, but list "
             "numeric UIDs and GIDs\n");
+    fprintf(stderr,"      --all-times            display atime, mtime,"
+            " and ctime information\n");
     fprintf(stderr,"  -o                         like -l, but do not "
             "list group information\n");
@@ -867 +927 @@
     char current_dir[PVFS_NAME_MAX] = {0};
     int found_one = 0;
+    char *entry_buffer = malloc(ENTRY_MAX);
 
     process_name = argv[0];
-
     user_opts = parse_args(argc, argv);
     if (!user_opts)
@@ -960 +1020 @@
         else /* Root directory case has nothing to match */
         {
-            substr = &user_opts->start[i][strlen(user_opts->start[i])-1];
+            substr = &user_opts->start[i][strlen(user_opts->start[i])];
         }
 
@@ -972 +1032 @@
         user_opts->start[i][++j] = '\0';
 
-        do_list(user_opts->start[i], pvfs_path[i], fs_id_array[i], user_opts);
+        do_list(user_opts->start[i], pvfs_path[i], fs_id_array[i], user_opts, entry_buffer);
 
         if (user_opts->num_starts > 1)
@@ -982 +1042 @@
     PVFS_sys_finalize();
     free(user_opts);
+    free(entry_buffer);
 
     return(ret);

branches/Orange-Branch/src/apps/admin/pvfs2-stat.c

@@ -55 +55 @@
 {
    int               ret          = -1,
+                     ret_agg      =  0,
                      i            =  0;
    char           ** ppszPvfsPath = NULL;
@@ -146 +147 @@
          fprintf(stderr, "Error stating [%s]\n", user_opts.pszFiles[i]);
       }
+      ret_agg |= ret;
    }
 
@@ -174 +176 @@
    }
 
-   return(0);
+   return(ret_agg);
 }
 

branches/Orange-Branch/src/apps/fuse/pvfs2fuse.c

@@ -10 +10 @@
  *           Sumit Narayan
  *
- *   $Date: 2009-03-06 13:54:32 $
- *   $Revision: 1.3 $
+ *   $Date: 2010-04-30 20:00:42 $
+ *   $Revision: 1.3.8.1 $
  *
  *   Documentation: http://www.engr.uconn.edu/~sun03001/docs/pvfs2fuse-rpt.pdf
  */
 
-/* char *pvfs2fuse_version = "$Id: pvfs2fuse.c,v 1.3 2009-03-06 13:54:32 slang Exp $"; */
+/* char *pvfs2fuse_version = "$Id: pvfs2fuse.c,v 1.3.8.1 2010-04-30 20:00:42 bligon Exp $"; */
 char *pvfs2fuse_version = "0.01";
 
@@ -246 +246 @@
    pvfs_fuse_handle_t   pfh;
 
-/* sumit */
-   system("echo 1 >> /tmp/abc");
-
    ret = lookup( path, &pfh, PVFS2_LOOKUP_LINK_NO_FOLLOW );
    if ( ret < 0 )
@@ -646 +643 @@
    {
           PVFS_Request_free(&mem_req);
-/* sumit */
-   ret = PVFS_sys_flush(pfh->ref, &pfh->creds);
-
-   if(ret < 0)
-          return PVFS_ERROR_TO_ERRNO_N(ret);
-/* end sumit */
           return(resp_io.total_completed);
    }
@@ -716 +707 @@
    (void) isdatasync;
    (void) fi;
-
-/* sumit */
-   int ret;
-   pvfs_fuse_handle_t pfh;
-
-   ret = lookup(path, &pfh, PVFS2_LOOKUP_LINK_FOLLOW);
-   if(ret < 0)
-          return PVFS_ERROR_TO_ERRNO_N(ret);
-
-   ret = PVFS_sys_flush(pfh.ref, &pfh.creds);
-
-   if(ret < 0)
-          return PVFS_ERROR_TO_ERRNO_N(ret);
-/* end sumit */
 
    return 0;

branches/Orange-Branch/src/apps/kernel/linux/pvfs2-client-core.c

@@ -131 +131 @@
   for the blocking ioctl.
 */
+#define REMOUNT_NOTCOMPLETED    0
+#define REMOUNT_COMPLETED       1
+#define REMOUNT_FAILED          2
 static pthread_t remount_thread;
 static pthread_mutex_t remount_mutex = PTHREAD_MUTEX_INITIALIZER;
-static int remount_complete = 0;
+static int remount_complete = REMOUNT_NOTCOMPLETED;
+
 
 /* used for generating unique dynamic mount point names */
@@ -503 +507 @@
       upcalls for each fs mounted by the kernel at this point
      */
+
+    /* if PINT_dev_remount fails set remount_complete appropriately */
     if (PINT_dev_remount())
     {
         gossip_err("*** Failed to remount filesystems!\n");
-    }
-
-    remount_complete = 1;
+        remount_complete = REMOUNT_FAILED;
+    }
+    else
+    {
+        remount_complete = REMOUNT_COMPLETED;
+    }
     pthread_mutex_unlock(&remount_mutex);
 
@@ -2843 +2852 @@
     }
 
-    if (!remount_complete &&
-        (vfs_request->in_upcall.type != PVFS2_VFS_OP_FS_MOUNT))
+    if (remount_complete == REMOUNT_NOTCOMPLETED &&
+        (vfs_request->in_upcall.type != PVFS2_VFS_OP_FS_MOUNT) && 
+        (vfs_request->in_upcall.type != PVFS2_VFS_OP_CANCEL) )
     {
         gossip_debug(
@@ -3124 +3134 @@
         {
             vfs_request = vfs_request_array[i];
+
             assert(vfs_request);
 /*             assert(vfs_request->op_id == op_id_array[i]); */
@@ -3270 +3281 @@
             assert(ret == 0);
         }
+
+        /* The status of the remount thread needs to be checked in the event 
+         * the remount fails on client-core startup. If this is the initial 
+         * startup then any mount requests will fail as expected and the 
+         * client-core will behave normally. However, if a mount was 
+         * previously successful (in a previous client-core incarnation) 
+         * client-core doesn't check if the remount succeeded before 
+         * handling the mount request and fs_add. Then any subsequent requests
+         * cause this thread spins around PINT_dev_test_unexpected.
+         *
+         * With the current structure of process_vfs_request, creating the 
+         * remount thread before entering the while loop, it seems exiting 
+         * client-core on a failed remount attempt is the most staight forward 
+         * way to handle this case. Exiting will cause the parent to kickoff 
+         * another client-core and try the remount until it succeeds.
+         */
+        if( remount_complete == REMOUNT_FAILED )
+        {
+            gossip_debug(GOSSIP_CLIENTCORE_DEBUG,
+                         "%s: remount not completed successfully, no longer "
+                         "handling requests.\n", __func__);
+            return -PVFS_EAGAIN; 
+        }
     }
 
@@ -3580 +3614 @@
     if (ret)
     {
-        gossip_err("Failed to process vfs requests!");
+        gossip_err("Failed to process vfs requests!\n");
     }
 
     /* join remount thread; should be long done by now */
-    if (remount_complete)
+    if (remount_complete == REMOUNT_COMPLETED )
     {
         pthread_join(remount_thread, NULL);
@@ -3619 +3653 @@
        PINT_sys_release(static_acache_timer_sm_p->sys_op_id);
 
+    gossip_err("pvfs2-client-core shutting down.\n");
     if (PVFS_sys_finalize())
     {
-        gossip_err("Failed to finalize PVFS\n");
         return 1;
+    }
+
+    /* if failed remount tell the parent it's something we did wrong. */
+    if( remount_complete != REMOUNT_COMPLETED )
+    {
+        return(-PVFS_EAGAIN);
     }
 
@@ -3631 +3671 @@
     }
 
-    gossip_debug(GOSSIP_CLIENTCORE_DEBUG, "%s terminating\n", argv[0]);
     return 0;
 }

branches/Orange-Branch/src/apps/kernel/linux/pvfs2-client.c

@@ -263 +263 @@
                     fprintf(stderr, "Device error caught, exiting now...\n");
                     exit(1);        
+                }
+
+                /* catch special case of exiting due to inability to remount */
+                /* we want to try again in this case. */
+                if (WEXITSTATUS(ret) == (unsigned char)-PVFS_EAGAIN)
+                {
+                    continue;
                 }
 

branches/Orange-Branch/src/client/sysint/client-state-machine.c

@@ -154 +154 @@
             if (user_ptr_array)
             {
-                user_ptr_array[i] = (void *)sm_p->user_ptr;
+                /* if this smcb has been set cancelled and is a PVFS_SYS_IO
+                 * state machine then stick the user_ptr of the base frame
+                 * in to the user_ptr_array instead of the standard sm_p 
+                 * user_ptr. This prevents segfaults back in 
+                 * process_vfs_requests which expects the pointer to be a 
+                 * vfs_request.
+                 */
+                if( smcb->op_cancelled && smcb->op == PVFS_SYS_IO )
+                {
+                    PINT_client_sm *sm_base_p = PINT_sm_frame(smcb,
+                                                 (-(smcb->frame_count -1)));
+                    assert(sm_base_p);
+                    gossip_debug(GOSSIP_CANCEL_DEBUG, "%s: assignment of "
+                                 "PVFS_SYS_IO user_ptr from sm_base_p(%p), "
+                                 "user_ptr(%p)\n", __func__, sm_base_p, 
+                                 sm_base_p->user_ptr);
+                    user_ptr_array[i] = sm_base_p->user_ptr;
+                }
+                else
+                {
+                    user_ptr_array[i] = (void *)sm_p->user_ptr;
+                }
             }
             s_completion_list[i] = NULL;
@@ -188 +209 @@
     */
     assert(sm_p);
+    
+    PINT_client_sm *sm_base_p = 
+        PINT_sm_frame(smcb, (-(smcb->frame_count -1)));
+
+    assert(sm_base_p);
 
     /*
@@ -193 +219 @@
       completed and no job cancellation were issued at I/O cancel time
     */
-    if (sm_p->u.io.total_cancellations_remaining > 0)
-    {
-        sm_p->u.io.total_cancellations_remaining--;
+    if (sm_base_p->u.io.total_cancellations_remaining > 0)
+    {
+        sm_base_p->u.io.total_cancellations_remaining--;
     }
 
     gossip_debug(
         GOSSIP_IO_DEBUG, "(%p) cancelled_io_jobs_are_pending: %d "
-        "remaining (op %s)\n", sm_p,
-        sm_p->u.io.total_cancellations_remaining,
+        "remaining (op %s)\n", sm_base_p,
+        sm_base_p->u.io.total_cancellations_remaining,
         (PINT_smcb_complete(smcb) ? "complete" : "NOT complete"));
 
-    return (sm_p->u.io.total_cancellations_remaining != 0);
+    return (sm_base_p->u.io.total_cancellations_remaining != 0);
 }
 
@@ -313 +339 @@
                  "client_state_machine_terminate smcb %p\n",smcb);
 
-    PINT_EVENT_END(PINT_client_sys_event_id, pint_client_pid, NULL, sm_p->event_id, 0);
-
-    PVFS_hint_free(sm_p->hints);
-
     if (!((PINT_smcb_op(smcb) == PVFS_SYS_IO) &&
             (PINT_smcb_cancelled(smcb)) &&
@@ -322 +344 @@
         !PINT_smcb_immediate_completion(smcb))
     {
+        gossip_debug(GOSSIP_CLIENT_DEBUG,
+                 "client_state_machine_terminate smcb %p completing\n",smcb);
+
+        PINT_EVENT_END(PINT_client_sys_event_id, pint_client_pid, NULL, sm_p->event_id, 0);
+        PVFS_hint_free(sm_p->hints);
+        sm_p->hints = NULL;
+
         gossip_debug(GOSSIP_CLIENT_DEBUG, 
                 "add smcb %p to completion list\n", smcb);
         ret = add_sm_to_completion_list(smcb);
         assert(ret == 0);
+    }
+    else
+    {
+        gossip_debug(GOSSIP_CLIENT_DEBUG,
+                 "client_state_machine_terminate smcb %p waiting for cancelled jobs\n",smcb);
     }
     return SM_ACTION_TERMINATE;
@@ -506 +540 @@
         return 0;
     }
-   
+    
     /* We also don't cancel small I/O operations as posted by
      * sys-small-io.sm.  Check the corresponding flag.  We have 
      * to jump to the base frame rather than the current frame for this
      * information because small-io may have pushed a msgpairarray.
+     *
+     * sm_base_p is used below instead of sm_p since it contains the correct
+     * counters and context pointers. In the event the control block only
+     * has one frame it behaves as it did previously. If the cancellation is 
+     * occuring when a non-IO frame has been pushed on the stack, which doesn't
+     * have the expected structure, it doesn't cause a segfault but leaves
+     * it on the state machines stack.
      */ 
     sm_base_p = PINT_sm_frame(smcb, (-(smcb->frame_count -1)));
+    assert(sm_base_p);
     if(sm_base_p->u.io.small_io)
     {
@@ -532 +574 @@
 
     /* now run through and cancel the outstanding jobs */
-    for(i = 0; i < sm_p->u.io.context_count; i++)
-    {
-        PINT_client_io_ctx *cur_ctx = &sm_p->u.io.contexts[i];
+    for(i = 0; i < sm_base_p->u.io.context_count; i++)
+    {
+        PINT_client_io_ctx *cur_ctx = &sm_base_p->u.io.contexts[i];
         assert(cur_ctx);
 
@@ -550 +592 @@
                 break;
             }
-            sm_p->u.io.total_cancellations_remaining++;
+            sm_base_p->u.io.total_cancellations_remaining++;
         }
 
@@ -566 +608 @@
                 break;
             }
-            sm_p->u.io.total_cancellations_remaining++;
+            sm_base_p->u.io.total_cancellations_remaining++;
         }
 
@@ -581 +623 @@
                 break;
             }
-            sm_p->u.io.total_cancellations_remaining++;
+            sm_base_p->u.io.total_cancellations_remaining++;
         }
 
@@ -597 +639 @@
                 break;
             }
-            sm_p->u.io.total_cancellations_remaining++;
+            sm_base_p->u.io.total_cancellations_remaining++;
         }
     }
     gossip_debug(GOSSIP_CANCEL_DEBUG, "(%p) Total cancellations "
-                 "remaining: %d\n", sm_p,
-                 sm_p->u.io.total_cancellations_remaining);
+                 "remaining: %d\n", sm_base_p,
+                 sm_base_p->u.io.total_cancellations_remaining);
     return ret;
 }

branches/Orange-Branch/src/client/sysint/client-state-machine.h

@@ -421 +421 @@
     PVFS_BMI_addr_t *addr_array;
     char **fs_config_bufs;
-    int32_t *fs_config_buf_size;
+    int *fs_config_buf_size;
+    int result_count; /* number of servers that actually responded */
+    int* result_indexes; /* index into fs_config_bufs of valid responses */
 };
 
@@ -700 +702 @@
     PVFS_MGMT_GET_DIRDATA_HANDLE   = 80,
     PVFS_SERVER_GET_CONFIG         = 200,
-    PVFS_SERVER_FETCH_CONFIG         = 201,
     PVFS_CLIENT_JOB_TIMER          = 300,
     PVFS_CLIENT_PERF_COUNT_TIMER   = 301,

branches/Orange-Branch/src/client/sysint/fs-add.sm

@@ -45 +45 @@
 extern job_context_id pint_client_sm_context;
 
+static int server_fetch_config_comp_fn(
+    void *v_p, struct PVFS_server_resp *resp_p, int i);
+
 %%
 
@@ -74 +77 @@
     {
         jump pvfs2_server_fetch_config_nested_sm;
+        default => compare_hashes;
+    }
+
+    state compare_hashes
+    {
+        run fs_add_compare_hashes;
         default => final_cleanup;
     }
@@ -81 +90 @@
         run fs_add_final_cleanup;
         default => terminate;
+    }
+}
+
+nested machine pvfs2_server_fetch_config_nested_sm
+{
+    state fetch_setup_msgpair
+    {
+        run server_fetch_config_setup_msgpair;
+        success => fetch_xfer_msgpair;
+        default => fetch_cleanup;
+    }
+
+    state fetch_xfer_msgpair
+    {
+        jump pvfs2_msgpairarray_sm;
+        default => fetch_cleanup;
+    }
+
+    state fetch_cleanup
+    {
+        run server_fetch_config_cleanup;
+        default => return;
     }
 }
@@ -321 +352 @@
         return SM_ACTION_COMPLETE;
     }
+
+    sm_p->fetch_config.result_indexes = (int*)calloc(count,
+        sizeof(int));
+    if(sm_p->fetch_config.result_indexes == NULL)
+    {
+        free(sm_p->fetch_config.addr_array);
+        sm_p->fetch_config.addr_array = NULL;
+        free(sm_p->fetch_config.fs_config_bufs);
+        sm_p->fetch_config.fs_config_bufs = NULL;
+        free(sm_p->fetch_config.fs_config_buf_size);
+        sm_p->fetch_config.fs_config_buf_size = NULL;
+        free(sm_p->fetch_config.result_indexes);
+        sm_p->fetch_config.result_indexes = NULL;
+        js_p->error_code = -PVFS_ENOMEM;
+        return SM_ACTION_COMPLETE;
+    }
+
     js_p->error_code = 0;
     return SM_ACTION_COMPLETE;
@@ -348 +396 @@
     size_t digest_len;
     PVFS_fs_id fsid;
-
-    assert(js_p->error_code == 0);
+    int* indexes = sm_p->fetch_config.result_indexes;
+
+    count = sm_p->fetch_config.result_count;
+    if(count < 2)
+    {
+        /* nothing to compare */
+        goto out;
+    }
+
     fsid = sm_p->u.get_config.mntent->fs_id;
-    count = sm_p->fetch_config.nservers;
     sha1_fs_digests = (char **) calloc(count, sizeof(char *));
     if (sha1_fs_digests == NULL)
@@ -360 +414 @@
     for (i = 1; i < count; i++)
     {
-        if (sm_p->fetch_config.fs_config_buf_size[0] != sm_p->fetch_config.fs_config_buf_size[i])
+        if (sm_p->fetch_config.fs_config_buf_size[indexes[0]] !=
+            sm_p->fetch_config.fs_config_buf_size[indexes[i]])
         {
             fs_conf_size_mismatch = 1;
@@ -371 +426 @@
         {
             gossip_err("     FS config file on %s -> (size) %d\n", 
-                   PVFS_mgmt_map_addr(fsid, sm_p->cred_p, sm_p->fetch_config.addr_array[i], &tmp),
-                   sm_p->fetch_config.fs_config_buf_size[i] - 1);
+                   PVFS_mgmt_map_addr(fsid, sm_p->cred_p,
+                   sm_p->fetch_config.addr_array[indexes[i]], &tmp),
+                   sm_p->fetch_config.fs_config_buf_size[indexes[i]] - 1);
             ret = -PVFS_EINVAL;
         }
@@ -379 +435 @@
     for (i = 0; i < count; i++)
     {
-        ret = PINT_util_digest_sha1(sm_p->fetch_config.fs_config_bufs[i], 
-                                    sm_p->fetch_config.fs_config_buf_size[i],
+        ret =
+        PINT_util_digest_sha1(sm_p->fetch_config.fs_config_bufs[indexes[i]], 
+                                    sm_p->fetch_config.fs_config_buf_size[indexes[i]],
                                     &sha1_fs_digests[i], &digest_len);
         if (ret < 0)
@@ -401 +458 @@
             hash2str((unsigned char *) sha1_fs_digests[i], digest_len, str);
             gossip_err("     FS config file on %s -> (SHA1) %s\n", 
-                   PVFS_mgmt_map_addr(fsid, sm_p->cred_p, sm_p->fetch_config.addr_array[i], &tmp), str);
+                   PVFS_mgmt_map_addr(fsid, sm_p->cred_p,
+                   sm_p->fetch_config.addr_array[indexes[i]], &tmp), str);
         }
         ret = -PVFS_EINVAL;
@@ -419 +477 @@
     free(sm_p->fetch_config.fs_config_bufs);
     sm_p->fetch_config.fs_config_bufs = NULL;
+    free(sm_p->fetch_config.result_indexes);
+    sm_p->fetch_config.result_indexes = NULL;
     free(sm_p->fetch_config.fs_config_buf_size);
     sm_p->fetch_config.fs_config_buf_size = NULL;
@@ -527 +587 @@
 }
 
+static PINT_sm_action fs_add_compare_hashes(
+    struct PINT_smcb *smcb, job_status_s *js_p)
+{
+    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+
+    if (js_p->error_code == SKIP_INTEGRITY_CHECK)
+        js_p->error_code = 0;
+    else
+    {
+        compare_hashes(sm_p, js_p);
+    }
+
+    return SM_ACTION_COMPLETE;
+}
+
 static PINT_sm_action fs_add_final_cleanup(
     struct PINT_smcb *smcb, job_status_s *js_p)
 {
     struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+    if(sm_p->u.get_config.free_config_flag)
+    {
+        PINT_config_release(sm_p->u.get_config.config);
+        free(sm_p->u.get_config.config);
+    }
     if (js_p->error_code == SKIP_INTEGRITY_CHECK)
         js_p->error_code = 0;
-    else if (js_p->error_code == 0)
-    {
-        compare_hashes(sm_p, js_p);
-    }
-    if(sm_p->u.get_config.free_config_flag)
-    {
-        PINT_config_release(sm_p->u.get_config.config);
-        free(sm_p->u.get_config.config);
-    }
     sm_p->error_code = js_p->error_code;
     PINT_SET_OP_COMPLETE;
     return SM_ACTION_TERMINATE;
+}
+
+static PINT_sm_action server_fetch_config_setup_msgpair(struct PINT_smcb *smcb,
+                                           job_status_s *js_p)
+{
+    int i;
+    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+    PINT_sm_msgpair_state *msg_p = NULL;
+
+    gossip_debug(GOSSIP_CLIENT_DEBUG,
+                "fetch_config state: server_fetch_config_setup_msgpair\n");
+
+    foreach_msgpair(&sm_p->msgarray_op, msg_p, i)
+    {
+        sm_p->msgarray_op.msgarray[i].enc_type = sm_p->msgarray_op.msgpair.enc_type;
+        PINT_SERVREQ_GETCONFIG_FILL(msg_p->req, *sm_p->cred_p, sm_p->hints);
+
+        msg_p->fs_id = PVFS_FS_ID_NULL;
+        msg_p->handle = PVFS_HANDLE_NULL;
+        /* only try once to retrieve a config file from each server */
+        msg_p->retry_flag = PVFS_MSGPAIR_NO_RETRY;
+        msg_p->comp_fn = server_fetch_config_comp_fn;
+        msg_p->svr_addr = sm_p->fetch_config.addr_array[i];
+    }
+    /* don't complain so much about servers we can't reach yet */
+    sm_p->msgarray_op.params.quiet_flag = 1; 
+
+    js_p->error_code = 0;
+
+    PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op);
+    return SM_ACTION_COMPLETE;
+}
+
+static PINT_sm_action server_fetch_config_cleanup(struct PINT_smcb *smcb,
+                                    job_status_s *js_p)
+{
+    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
+    PINT_msgpairarray_destroy(&sm_p->msgarray_op);
+    /* preserve js_p->error_code */
+
+    return SM_ACTION_COMPLETE;
+}
+
+static int server_fetch_config_comp_fn(
+    void *v_p,
+    struct PVFS_server_resp *resp_p,
+    int i)
+{
+    PINT_smcb *smcb = v_p;
+    PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM);
+    int j;
+
+    /* if this particular request was successful, then store the server
+     * response and let the caller sort it out */
+
+    if (sm_p->msgarray_op.msgarray[i].op_status == 0)
+    {
+        sm_p->fetch_config.fs_config_bufs[i] = 
+                strdup(resp_p->u.getconfig.fs_config_buf);
+        sm_p->fetch_config.fs_config_buf_size[i] = 
+                resp_p->u.getconfig.fs_config_buf_size;
+    }
+
+    /* is this this last response? */
+    if (i == (sm_p->msgarray_op.count -1))
+    {
+        /* look through responses, count the number of valid responses we
+         * received, and mark where they are in the array
+         */
+        sm_p->fetch_config.result_count = 0;
+        for(j=0; j<sm_p->msgarray_op.count; j++)
+        {
+            if(sm_p->msgarray_op.msgarray[j].op_status == 0)
+            {
+                sm_p->fetch_config.result_indexes[sm_p->fetch_config.result_count]
+                    = j;
+                sm_p->fetch_config.result_count++;
+            }
+        }
+        if(sm_p->fetch_config.result_count > 0)
+        {
+            /* we got at least one config file */
+            return(0);
+        }
+        else
+        {
+            /* pick an error code */
+            return PINT_msgarray_status(&sm_p->msgarray_op);
+        }
+    }
+    return 0;
 }
 

branches/Orange-Branch/src/client/sysint/initialize.c

@@ -128 +128 @@
         goto error_exit;
     }
+    client_status_flag |= CLIENT_DIST_INIT;
     
     /* initlialize the protocol encoder */

branches/Orange-Branch/src/client/sysint/server-get-config.sm

@@ -28 +28 @@
     void *v_p, struct PVFS_server_resp *resp_p, int i);
 
-static int server_fetch_config_comp_fn(
-    void *v_p, struct PVFS_server_resp *resp_p, int i);
-
 %%
 
@@ -73 +70 @@
     {
         run server_get_config_parent_cleanup;
-        default => terminate;
-    }
-}
-
-nested machine pvfs2_server_fetch_config_nested_sm
-{
-    state fetch_setup_msgpair
-    {
-        run server_fetch_config_setup_msgpair;
-        success => fetch_xfer_msgpair;
-        default => fetch_cleanup;
-    }
-
-    state fetch_xfer_msgpair
-    {
-        jump pvfs2_msgpairarray_sm;
-        default => fetch_cleanup;
-    }
-
-    state fetch_cleanup
-    {
-        run server_fetch_config_cleanup;
-        default => return;
-    }
-}
-
-machine pvfs2_server_fetch_config_sm
-{
-    state run_fetch_nested 
-    {
-        jump pvfs2_server_fetch_config_nested_sm;
-        default => parent_fetch_cleanup;
-    }
-
-    state parent_fetch_cleanup
-    {
-        run server_fetch_config_parent_cleanup;
         default => terminate;
     }
@@ -407 +367 @@
     return 0;
 }
-
-/*
- * Force a fetch the configuration file from a given set of servers
- * This is different from get_config() in that we do not depend on the
- * configuration server specified in the mnt tab file
- * Assumes that servers specified in the array are unique!
- * CURRENTLY UNUSED
- */
-#ifdef __GNUC__
-static int PINT_fetch_config_list(int nservers, PVFS_BMI_addr_t *servers,
-                char **fs_config_bufs, int *fs_config_buf_size) __attribute__((unused));
-#endif
-
-static int PINT_fetch_config_list(int nservers, PVFS_BMI_addr_t *servers,
-                char **fs_config_bufs, int *fs_config_buf_size)
-{
-    int ret = -PVFS_EINVAL;
-    PINT_smcb *smcb = NULL;
-    PINT_client_sm *sm_p = NULL;
-    PVFS_error error = 0;
-    PVFS_credentials creds;
-    PVFS_sys_op_id op_id;
-
-    gossip_debug(GOSSIP_CLIENT_DEBUG,
-                 "PINT_fetch_config_list entered\n");
-    if (nservers < 0 || !servers || !fs_config_bufs || !fs_config_buf_size)
-    {
-        return ret;
-    }
-
-    PVFS_util_gen_credentials(&creds);
-    PINT_smcb_alloc(&smcb, PVFS_SERVER_FETCH_CONFIG,
-             sizeof(struct PINT_client_sm),
-             client_op_state_get_machine,
-             client_state_machine_terminate,
-             pint_client_sm_context);
-    if (smcb == NULL)
-    {
-        return -PVFS_ENOMEM;
-    }
-    sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
-
-    /* NOTE: we set these fields manually here rather than use
-     * PINT_init_msgarray_params(), because we don't yet have a server
-     * configuration file to override default parameters.
-     */
-    sm_p->msgarray_op.params.job_context = pint_client_sm_context;
-    sm_p->msgarray_op.params.job_timeout = 30;   /* 30 second job timeout */
-    sm_p->msgarray_op.params.retry_delay = 2000; /* 2 second retry delay */
-    sm_p->msgarray_op.params.retry_limit = 5;    /* retry up to 5 times */
-    PINT_init_sysint_credentials(sm_p->cred_p, &creds);
-    sm_p->fetch_config.nservers = nservers;
-    sm_p->fetch_config.addr_array = servers;
-    sm_p->fetch_config.fs_config_bufs = fs_config_bufs;
-    sm_p->fetch_config.fs_config_buf_size = fs_config_buf_size;
-
-    ret = PINT_msgpairarray_init(&sm_p->msgarray_op, nservers);
-    if(ret != 0)
-    {
-        PINT_smcb_free(smcb);
-        return ret;
-    }
-
-    ret = PINT_client_state_machine_post(
-        smcb, &op_id, NULL);
-    if (ret)
-    {
-        PVFS_perror_gossip("PINT_client_state_machine_post call", ret);
-        error = ret;
-    }
-    else
-    {
-        ret = PVFS_sys_wait(op_id, "X-fetch_config", &error);
-        if (ret)
-        {
-            PVFS_perror_gossip("PVFS_sys_wait call", ret);
-            error = ret;
-        }
-    }
-
-    PINT_sys_release(op_id);
-    return(error);
-}
-
-static PINT_sm_action server_fetch_config_setup_msgpair(struct PINT_smcb *smcb,
-                                           job_status_s *js_p)
-{
-    int i;
-    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
-    PINT_sm_msgpair_state *msg_p = NULL;
-
-    gossip_debug(GOSSIP_CLIENT_DEBUG,
-                "fetch_config state: server_fetch_config_setup_msgpair\n");
-
-    foreach_msgpair(&sm_p->msgarray_op, msg_p, i)
-    {
-        sm_p->msgarray_op.msgarray[i].enc_type = sm_p->msgarray_op.msgpair.enc_type;
-        PINT_SERVREQ_GETCONFIG_FILL(msg_p->req, *sm_p->cred_p, sm_p->hints);
-
-        msg_p->fs_id = PVFS_FS_ID_NULL;
-        msg_p->handle = PVFS_HANDLE_NULL;
-        msg_p->retry_flag = PVFS_MSGPAIR_RETRY;
-        msg_p->comp_fn = server_fetch_config_comp_fn;
-        msg_p->svr_addr = sm_p->fetch_config.addr_array[i];
-    }
-
-    js_p->error_code = 0;
-
-    PINT_sm_push_frame(smcb, 0, &sm_p->msgarray_op);
-    return SM_ACTION_COMPLETE;
-}
-
-static PINT_sm_action server_fetch_config_cleanup(struct PINT_smcb *smcb,
-                                    job_status_s *js_p)
-{
-    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
-    PINT_msgpairarray_destroy(&sm_p->msgarray_op);
-    /* preserve js_p->error_code */
-
-    return SM_ACTION_COMPLETE;
-}
-
-static int server_fetch_config_comp_fn(
-    void *v_p,
-    struct PVFS_server_resp *resp_p,
-    int i)
-{
-    PINT_smcb *smcb = v_p;
-    PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_MSGPAIR_PARENT_SM);
-
-    /* if this particular request was successful, then store the server
-     * response and let the caller sort it out */
-
-    if (sm_p->msgarray_op.msgarray[i].op_status == 0)
-    {
-        sm_p->fetch_config.fs_config_bufs[i] = 
-                strdup(resp_p->u.getconfig.fs_config_buf);
-        sm_p->fetch_config.fs_config_buf_size[i] = 
-                resp_p->u.getconfig.fs_config_buf_size;
-    }
-
-    /* if this is the last response, check all of the status values
-     * and return error code if any requests failed
-     */
-    if (i == (sm_p->msgarray_op.count -1))
-    {
-        return PINT_msgarray_status(&sm_p->msgarray_op);
-    }
-    return 0;
-}
-
-static PINT_sm_action server_fetch_config_parent_cleanup(struct PINT_smcb *smcb,
-                                    job_status_s *js_p)
-{
-    struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
-    sm_p->error_code  = js_p->error_code;
-    PINT_SET_OP_COMPLETE;
-    return SM_ACTION_DEFERRED;
-}
         
 /*

branches/Orange-Branch/src/client/sysint/sys-create.sm

@@ -803 +803 @@
      */
 
-    if(sm_p->u.create.user_requested_num_data_files > 1)
+    if(sm_p->u.create.user_requested_num_data_files > 0)
     {
         /* specified by sys_create caller */

branches/Orange-Branch/src/client/sysint/sys-io.sm

@@ -515 +515 @@
                 sm_p->object_ref.handle,
                 IO_ATTR_MASKS,
-//                0,
                 sm_p->hints);
     }

branches/Orange-Branch/src/client/sysint/sys-truncate.sm

@@ -154 +154 @@
         PVFS_ATTR_META_ALL|PVFS_ATTR_COMMON_TYPE,
         PVFS_TYPE_METAFILE,
-        0);
+        PINT_SM_GETATTR_BYPASS_CACHE);
 
     return PINT_client_state_machine_post(
@@ -314 +314 @@
     struct PINT_client_sm *sm_p = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
 
+    /* check for write access */
+    js_p->error_code = PINT_check_mode(
+        &sm_p->getattr.attr, sm_p->cred_p->uid, sm_p->cred_p->gid, PINT_ACCESS_WRITABLE);
+    if(js_p->error_code)
+    {
+        return 1;
+    }
+
     /* determine if we need to unstuff or not to service this request */
     if(unstuff_needed(

branches/Orange-Branch/src/common/gossip/gossip.c

@@ -480 +480 @@
             gettimeofday(&tv, 0);
             tp = tv.tv_sec;
-            strftime(bptr, 14, "%m/%d %H:%M] ", localtime(&tp));
-            bptr += 13;
-            bsize -= 13;
+            strftime(bptr, 22, "%m/%d/%Y %H:%M:%S] ", localtime(&tp));
+            bptr += 21;
+            bsize -= 21;
             break;
         case GOSSIP_LOGSTAMP_THREAD:

branches/Orange-Branch/src/common/misc/pint-hint.c

@@ -565 +565 @@
     while(h)
     {
-        if(!strcmp(hint->type_string, name))
+        if(!strcmp(h->type_string, name))
         {
             if(length)

branches/Orange-Branch/src/common/misc/pint-util.c

@@ -18 +18 @@
 #include <unistd.h>
 
+#include <grp.h>
+#include <pwd.h>
+#include <sys/types.h>
+
 #define __PINT_REQPROTO_ENCODE_FUNCS_C
 #include "gen-locks.h"
@@ -24 +28 @@
 #include "gossip.h"
 #include "pvfs2-req-proto.h"
+
+#include "pvfs2-debug.h"
+#include "bmi-byteswap.h"
+
+#ifdef HAVE_GETPWUID
+static gen_mutex_t check_group_mutex = GEN_MUTEX_INITIALIZER;
+static int pw_buf_size = 1024;      // 1 KB
+static int gr_buf_size = 1024*1024; // 1 MB
+static char* check_group_pw_buffer = NULL;
+static char* check_group_gr_buffer = NULL;
+#endif
+static int PINT_check_group(uid_t uid, gid_t gid);
 
 void PINT_time_mark(PINT_time_marker *out_marker)
@@ -542 +558 @@
 }
 
+/* PINT_check_mode()
+ *
+ * checks to see if the type of access described by "access_type" is permitted 
+ * for user "uid" of group "gid" on the object with attributes "attr"
+ *
+ * returns 0 on success, -PVFS_EACCES if permission is not granted
+ */
+int PINT_check_mode(
+    PVFS_object_attr *attr,
+    PVFS_uid uid, PVFS_gid gid,
+    enum PINT_access_type access_type)
+{
+    int in_group_flag = 0;
+    int ret = 0;
+
+    /* if we don't have masks for the permission information that we
+     * need, then the system is broken
+     */
+    assert(attr->mask & PVFS_ATTR_COMMON_UID &&
+           attr->mask & PVFS_ATTR_COMMON_GID &&
+           attr->mask & PVFS_ATTR_COMMON_PERM);
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - check_mode called --- "
+                 "(uid=%d,gid=%d,access_type=%d)\n", uid, gid, access_type);
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - object attributes --- "
+                 "(uid=%d,gid=%d,mode=%d)\n", attr->owner, attr->group,
+                 attr->perms);
+
+    /* give root permission, no matter what */
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG,
+                 " - checking if uid (%d) is root ...\n", uid);
+    if (uid == 0)
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        return 0;
+    }
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+
+    /* see if uid matches object owner */
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking if owner (%d) "
+        "matches uid (%d)...\n", attr->owner, uid);
+    if(attr->owner == uid)
+    {
+        /* see if object user permissions match access type */
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking if permissions "
+            "(%d) allows access type (%d) for user...\n", attr->perms, access_type);
+        if(access_type == PINT_ACCESS_READABLE && (attr->perms &
+            PVFS_U_READ))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        if(access_type == PINT_ACCESS_WRITABLE && (attr->perms &
+            PVFS_U_WRITE))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        if(access_type == PINT_ACCESS_EXECUTABLE && (attr->perms &
+            PVFS_U_EXECUTE))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+    }
+    else
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+    }
+
+    /* see if other bits allow access */
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking if permissions "
+        "(%d) allows access type (%d) by others...\n", attr->perms, access_type);
+    if(access_type == PINT_ACCESS_READABLE && (attr->perms &
+        PVFS_O_READ))
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        return(0);
+    }
+    if(access_type == PINT_ACCESS_WRITABLE && (attr->perms &
+        PVFS_O_WRITE))
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        return(0);
+    }
+    if(access_type == PINT_ACCESS_EXECUTABLE && (attr->perms &
+        PVFS_O_EXECUTE))
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        return(0);
+    }
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+
+    /* see if gid matches object group */
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking if group (%d) "
+        "matches gid (%d)...\n", attr->group, gid);
+    if(attr->group == gid)
+    {
+        /* default group match */
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+        in_group_flag = 1;
+    }
+    else
+    {
+        /* no default group match, check supplementary groups */
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking for"
+            " supplementary group match...\n");
+        ret = PINT_check_group(uid, attr->group);
+        if(ret == 0)
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            in_group_flag = 1;
+        }
+        else
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+            if(ret != -PVFS_ENOENT)
+            {
+                /* system error; not just failed match */
+                return(ret);
+            }
+        }
+    }
+
+    if(in_group_flag)
+    {
+        /* see if object group permissions match access type */
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - checking if permissions "
+            "(%d) allows access type (%d) for group...\n", attr->perms, access_type);
+        if(access_type == PINT_ACCESS_READABLE && (attr->perms &
+            PVFS_G_READ))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        if(access_type == PINT_ACCESS_WRITABLE && (attr->perms &
+            PVFS_G_WRITE))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        if(access_type == PINT_ACCESS_EXECUTABLE && (attr->perms &
+            PVFS_G_EXECUTE))
+        {
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - yes\n");
+            return(0);
+        }
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, " - no\n");
+    }
+  
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "******PINT_check_mode: denying access\n");
+    /* default case: access denied */
+    return -PVFS_EACCES;
+}
+
+/* PINT_check_group()
+ *
+ * checks to see if uid is a member of gid
+ * 
+ * returns 0 on success, -PVFS_ENOENT if not a member, other PVFS error codes
+ * on system failure
+ */
+static int PINT_check_group(uid_t uid, gid_t gid)
+{
+#ifdef HAVE_GETPWUID
+    struct passwd pwd;
+    struct passwd* pwd_p = NULL;
+    struct group grp;
+    struct group* grp_p = NULL;
+    int i = 0;
+    int ret = -1;
+
+    /* Explanation: 
+     *
+     * We use the _r variants of getpwuid and getgrgid in order to insure
+     * thread safety; particularly if this function ever gets called in a
+     * client side situation in which we can't prevent the application from
+     * making conflicting calls.
+     *
+     * These _r functions require that a buffer be supplied for the user and
+     * group information, however.  These buffers may be unconfortably large
+     * for the stack, so we malloc them on a static pointer and then mutex
+     * lock this function so that it can still be reentrant.
+     */
+
+    gen_mutex_lock(&check_group_mutex);
+
+    if(!check_group_pw_buffer)
+    {
+        check_group_pw_buffer = (char*)malloc(pw_buf_size);
+        check_group_gr_buffer = (char*)malloc(gr_buf_size);
+        if(!check_group_pw_buffer || !check_group_gr_buffer)
+        {
+            if(check_group_pw_buffer)
+            {
+                free(check_group_pw_buffer);
+                check_group_pw_buffer = NULL;
+            }
+            if(check_group_gr_buffer)
+            {
+                free(check_group_gr_buffer);
+                check_group_gr_buffer = NULL;
+            }
+            gen_mutex_unlock(&check_group_mutex);
+            return(-PVFS_ENOMEM);
+        }
+    }
+
+    /* get user information */
+    ret = getpwuid_r(uid, &pwd, check_group_pw_buffer, pw_buf_size, &pwd_p);
+    if(ret != 0 || pwd_p == NULL)
+    {
+        gen_mutex_unlock(&check_group_mutex);
+        gossip_err("Get user info for (uid=%d) failed."
+                   "errno [%d] error_msg [%s]\n",
+                   uid, ret, strerror(ret));
+        return(-PVFS_EINVAL);
+    }
+
+    /* check primary group */
+    if(pwd.pw_gid == gid)
+    {
+        gen_mutex_unlock(&check_group_mutex);
+        return 0;
+    }
+
+    /* get the members of the group */
+    ret = getgrgid_r(gid, &grp, check_group_gr_buffer, gr_buf_size, &grp_p);
+    if(ret != 0 || grp_p == NULL)
+    {
+      gen_mutex_unlock(&check_group_mutex);
+      gossip_err("Get members for group (gid=%d) failed."
+                 "errno [%d] error_msg [%s]\n",
+                 gid, ret, strerror(ret));
+      return(-PVFS_EINVAL);
+    }
+
+    for(i = 0; grp.gr_mem[i] != NULL; i++)
+    {
+        if(0 == strcmp(pwd.pw_name, grp.gr_mem[i]))
+        {
+            gen_mutex_unlock(&check_group_mutex);
+            return 0;
+        } 
+    }
+
+    gen_mutex_unlock(&check_group_mutex);
+    return(-PVFS_ENOENT);
+#else
+    return 0;
+#endif
+}
+
+/* Checks if a given user is part of any groups that matches the file gid */
+static int in_group_p(PVFS_uid uid, PVFS_gid gid, PVFS_gid attr_group)
+{
+    if (attr_group == gid)
+        return 1;
+    if (PINT_check_group(uid, attr_group) == 0)
+        return 1;
+    return 0;
+}
+
+/*
+ * Return 0 if requesting clients is granted want access to the object
+ * by the acl. Returns -PVFS_E... otherwise.
+ */
+int PINT_check_acls(void *acl_buf, size_t acl_size, 
+    PVFS_object_attr *attr,
+    PVFS_uid uid, PVFS_gid gid, int want)
+{
+    pvfs2_acl_entry pe, *pa;
+    int i = 0, found = 0, count = 0;
+    assert(attr->mask & PVFS_ATTR_COMMON_UID &&
+           attr->mask & PVFS_ATTR_COMMON_GID &&
+           attr->mask & PVFS_ATTR_COMMON_PERM);
+
+    if (acl_size == 0)
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "no acl's present.. denying access\n");
+        return -PVFS_EACCES;
+    }
+
+    /* keyval for ACLs includes a \0. so subtract the thingie */
+    acl_size--;
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "PINT_check_acls: read keyval size "
+    " %d (%d acl entries)\n",
+        (int) acl_size, 
+        (int) (acl_size / sizeof(pvfs2_acl_entry)));
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "uid = %d, gid = %d, want = %d\n",
+        uid, gid, want);
+
+    assert(acl_buf);
+    /* if the acl format doesn't look valid, then return an error rather than
+     * asserting; we don't want the server to crash due to an invalid keyval
+     */
+    if((acl_size % sizeof(pvfs2_acl_entry)) != 0)
+    {
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "invalid acls on object\n");
+        return(-PVFS_EACCES);
+    }
+    count = acl_size / sizeof(pvfs2_acl_entry);
+
+    for (i = 0; i < count; i++)
+    {
+        pa = (pvfs2_acl_entry *) acl_buf + i;
+        /* 
+           NOTE: Remember that keyval is encoded as lebf, so convert it 
+           to host representation 
+        */
+        pe.p_tag  = bmitoh32(pa->p_tag);
+        pe.p_perm = bmitoh32(pa->p_perm);
+        pe.p_id   = bmitoh32(pa->p_id);
+        pa = &pe;
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "Decoded ACL entry %d "
+            "(p_tag %d, p_perm %d, p_id %d)\n",
+            i, pa->p_tag, pa->p_perm, pa->p_id);
+        switch(pa->p_tag) 
+        {
+            case PVFS2_ACL_USER_OBJ:
+                /* (May have been checked already) */
+                if (attr->owner == uid)
+                    goto check_perm;
+                break;
+            case PVFS2_ACL_USER:
+                if (pa->p_id == uid)
+                    goto mask;
+                break;
+            case PVFS2_ACL_GROUP_OBJ:
+                if (in_group_p(uid, gid, attr->group)) 
+                {
+                    found = 1;
+                    if ((pa->p_perm & want) == want)
+                        goto mask;
+                }
+                break;
+            case PVFS2_ACL_GROUP:
+                if (in_group_p(uid, gid, pa->p_id)) {
+                    found = 1;
+                    if ((pa->p_perm & want) == want)
+                        goto mask;
+                }
+                break;
+            case PVFS2_ACL_MASK:
+                break;
+            case PVFS2_ACL_OTHER:
+                if (found)
+                {
+                    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "(1) PINT_check_acls:"
+                        "returning access denied\n");
+                    return -PVFS_EACCES;
+                }
+                else
+                    goto check_perm;
+            default:
+                gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "(2) PINT_check_acls: "
+                        "returning EIO\n");
+                return -PVFS_EIO;
+        }
+    }
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "(3) PINT_check_acls: returning EIO\n");
+    return -PVFS_EIO;
+mask:
+    /* search the remaining entries */
+    i = i + 1;
+    for (; i < count; i++)
+    {
+        pvfs2_acl_entry me, *mask_obj = (pvfs2_acl_entry *) acl_buf + i;
+        
+        /* 
+          NOTE: Again, since pvfs2_acl_entry is in lebf, we need to
+          convert it to host endian format
+         */
+        me.p_tag  = bmitoh32(mask_obj->p_tag);
+        me.p_perm = bmitoh32(mask_obj->p_perm);
+        me.p_id   = bmitoh32(mask_obj->p_id);
+        mask_obj = &me;
+        gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "Decoded (mask) ACL entry %d "
+            "(p_tag %d, p_perm %d, p_id %d)\n",
+            i, mask_obj->p_tag, mask_obj->p_perm, mask_obj->p_id);
+        if (mask_obj->p_tag == PVFS2_ACL_MASK) 
+        {
+            if ((pa->p_perm & mask_obj->p_perm & want) == want)
+                return 0;
+            gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "(4) PINT_check_acls:"
+                "returning access denied (mask)\n");
+            return -PVFS_EACCES;
+        }
+    }
+
+check_perm:
+    if ((pa->p_perm & want) == want)
+        return 0;
+    gossip_debug(GOSSIP_PERMISSIONS_DEBUG, "(5) PINT_check_acls: returning"
+            "access denied\n");
+    return -PVFS_EACCES;
+}
+
 /*
  * Local variables:

branches/Orange-Branch/src/common/misc/pint-util.h

@@ -142 +142 @@
     PVFS_credentials *credentials);
 
+enum PINT_access_type
+{
+    PINT_ACCESS_EXECUTABLE = 1,
+    PINT_ACCESS_WRITABLE = 2,
+    PINT_ACCESS_READABLE = 4,
+};
+
+int PINT_check_mode(
+    PVFS_object_attr *attr,
+    PVFS_uid uid, PVFS_gid gid,
+    enum PINT_access_type access_type);
+
+int PINT_check_acls(void *acl_buf, size_t acl_size, 
+    PVFS_object_attr *attr,
+    PVFS_uid uid, PVFS_gid gid, int want);
+
 #endif /* __PINT_UTIL_H */
 

branches/Orange-Branch/src/common/misc/server-config.c

@@ -732 +732 @@
      * usec: [%H:%M:%S.%U]
      *
-     * datetime: [%m/%d %H:%M]
+     * datetime: [%m/%d/%Y %H:%M:%S]
      *
      * thread: [%H:%M:%S.%U (%lu)]

branches/Orange-Branch/src/common/misc/state-machine-fns.c

@@ -431 +431 @@
 {
     return smcb->op == PVFS_SERVER_GET_CONFIG 
-        || smcb->op == PVFS_SERVER_FETCH_CONFIG
         || smcb->op == PVFS_CLIENT_JOB_TIMER 
         || smcb->op == PVFS_CLIENT_PERF_COUNT_TIMER 

branches/Orange-Branch/src/io/bmi/bmi-method-support.h

@@ -59 +59 @@
 };
 
+/* flags that can be set per method to affect behavior */
+#define BMI_METHOD_FLAG_NO_POLLING 1
+
 /* This is the table of interface functions that must be provided by BMI
  * methods.
@@ -65 +68 @@
 {
     const char *method_name;
+    int flags;
     int (*initialize) (bmi_method_addr_p, int, int);
     int (*finalize) (void);

branches/Orange-Branch/src/io/bmi/bmi.c

@@ -81 +81 @@
 #ifdef __STATIC_METHOD_BMI_PORTALS__
 extern struct bmi_method_ops bmi_portals_ops;
+#endif
+#ifdef __STATIC_METHOD_BMI_ZOID__
+extern struct bmi_method_ops bmi_zoid_ops;
 #endif
 
@@ -99 +102 @@
     &bmi_portals_ops,
 #endif
+#ifdef __STATIC_METHOD_BMI_ZOID__
+    &bmi_zoid_ops,
+#endif
     NULL
 };
@@ -121 +127 @@
 
 static struct bmi_method_ops **active_method_table = NULL;
-static struct {
+
+struct method_usage_t {
     int iters_polled;  /* how many iterations since this method was polled */
     int iters_active;  /* how many iterations since this method had action */
     int plan;
-} *method_usage = NULL;
+    int flags;
+};
+
+static struct method_usage_t * expected_method_usage = NULL;
+static struct method_usage_t * unexpected_method_usage = NULL;
+
 static const int usage_iters_starvation = 100000;
 static const int usage_iters_active = 10000;
@@ -531 +543 @@
     known_method_count = 0;
 
-    if (method_usage)
-        free(method_usage);
+    if (expected_method_usage)
+        free(expected_method_usage);
+
+    if (unexpected_method_usage)
+       free(unexpected_method_usage);
 
     /* destroy the reference list */
@@ -914 +929 @@
  */
 static void
-construct_poll_plan(int nmeth, int *idle_time_ms)
+construct_poll_plan(struct method_usage_t * method_usage,
+      int nmeth, int *idle_time_ms)
 {
     int i, numplan;
@@ -923 +939 @@
         ++method_usage[i].iters_active;
         method_usage[i].plan = 0;
-        if (method_usage[i].iters_active <= usage_iters_active) {
+        if ((method_usage[i].iters_active <= usage_iters_active) &&
+            (!(method_usage[i].flags & BMI_METHOD_FLAG_NO_POLLING))){
             /* recently busy, poll */
             if (0) gossip_debug(GOSSIP_BMI_DEBUG_CONTROL,
@@ -991 +1008 @@
     *outcount = 0;
 
-    construct_poll_plan(tmp_active_method_count, &max_idle_time_ms);
+    construct_poll_plan(unexpected_method_usage,
+          tmp_active_method_count, &max_idle_time_ms);
 
     while (position < incount && i < tmp_active_method_count)
     {
-        if (method_usage[i].plan) {
+        if (unexpected_method_usage[i].plan) {
             ret = active_method_table[i]->testunexpected(
                 (incount - position), &tmp_outcount,
@@ -1007 +1025 @@
             position += tmp_outcount;
             (*outcount) += tmp_outcount;
-            method_usage[i].iters_polled = 0;
+            unexpected_method_usage[i].iters_polled = 0;
             if (ret)
-                method_usage[i].iters_active = 0;
+                unexpected_method_usage[i].iters_active = 0;
         }
         i++;
@@ -1088 +1106 @@
     }
 
-    construct_poll_plan(tmp_active_method_count, &max_idle_time_ms);
+    construct_poll_plan(expected_method_usage,
+          tmp_active_method_count, &max_idle_time_ms);
 
     while (position < incount && i < tmp_active_method_count)
     {
-        if (method_usage[i].plan) {
+        if (expected_method_usage[i].plan) {
             ret = active_method_table[i]->testcontext(
                 incount - position, 
@@ -1110 +1129 @@
             position += tmp_outcount;
             (*outcount) += tmp_outcount;
-            method_usage[i].iters_polled = 0;
+            expected_method_usage[i].iters_polled = 0;
             if (ret)
-                method_usage[i].iters_active = 0;
+                expected_method_usage[i].iters_active = 0;
         }
         i++;
@@ -2022 +2041 @@
 }
 
+
+/**
+ * Try to increase method_usage_t struct to include room for a new method.
+ */
+static int grow_method_usage (struct method_usage_t ** p, int newflags)
+{
+    struct method_usage_t * x = *p;
+    *p = malloc((active_method_count + 1) * sizeof(**p));
+    if (!*p) {
+        *p = x;
+        return 0;
+    }
+    if (active_method_count) {
+        memcpy(*p, x, active_method_count * sizeof(**p));
+        free(x);
+    }
+    memset(&((*p)[active_method_count]), 0, sizeof(**p));
+    (*p)[active_method_count].flags = newflags;
+
+    return 1;
+ }
+
 /*
  * Attempt to insert this name into the list of active methods,
@@ -2072 +2113 @@
     active_method_table[active_method_count] = meth;
 
-    x = method_usage;
-    method_usage = malloc((active_method_count + 1) * sizeof(*method_usage));
-    if (!method_usage) {
-        method_usage = x;
-        return -ENOMEM;
-    }
-    if (active_method_count) {
-        memcpy(method_usage, x, active_method_count * sizeof(*method_usage));
-        free(x);
-    }
-    memset(&method_usage[active_method_count], 0, sizeof(*method_usage));
+    if (!grow_method_usage (&unexpected_method_usage, meth->flags))
+       return -ENOMEM;
+
+    /**
+     * If we run out of memory here, the unexpected_method_usage will be
+     * larger than strictly required but there is no memory leak.
+     */
+
+    if (!grow_method_usage (&expected_method_usage, meth->flags))
+       return -ENOMEM;
 
     ++active_method_count;

branches/Orange-Branch/src/io/bmi/bmi_gm/bmi-gm.c

@@ -158 +158 @@
 const struct bmi_method_ops bmi_gm_ops = {
     .method_name = BMI_gm_method_name,
+    .flags = 0,
     .initialize = BMI_gm_initialize,
     .finalize = BMI_gm_finalize,

branches/Orange-Branch/src/io/bmi/bmi_ib/ib.c

@@ -23 +23 @@
 #include <src/common/gen-locks/gen-locks.h>  /* gen_mutex_t ... */
 #include <src/common/misc/pvfs2-internal.h>
+#include "pint-hint.h"
 
 #ifdef HAVE_VALGRIND_H
@@ -895 +896 @@
                  const void *buffer, bmi_size_t total_size,
                  enum bmi_buffer_type buffer_flag __unused,
-                 bmi_msg_tag_t tag, void *user_ptr, bmi_context_id context_id)
+                 bmi_msg_tag_t tag, void *user_ptr, bmi_context_id
+                 context_id, PVFS_hint hints __unused)
 {
     return post_send(id, remote_map, 0, &buffer, &total_size,
@@ -905 +907 @@
   const void *const *buffers, const bmi_size_t *sizes, int list_count,
   bmi_size_t total_size, enum bmi_buffer_type buffer_flag __unused,
-  bmi_msg_tag_t tag, void *user_ptr, bmi_context_id context_id)
+  bmi_msg_tag_t tag, void *user_ptr, bmi_context_id context_id, PVFS_hint
+  hints __unused)
 {
     return post_send(id, remote_map, list_count, buffers, sizes,
@@ -916 +919 @@
                            enum bmi_buffer_type buffer_flag __unused,
                            bmi_msg_tag_t tag, void *user_ptr,
-                           bmi_context_id context_id)
+                           bmi_context_id context_id, PVFS_hint hints
+                           __unused)
 {
     return post_send(id, remote_map, 0, &buffer, &total_size,
@@ -929 +933 @@
                                 enum bmi_buffer_type buffer_flag __unused,
                                 bmi_msg_tag_t tag, void *user_ptr,
-                                bmi_context_id context_id)
+                                bmi_context_id context_id, PVFS_hint hints
+                                __unused)
 {
     return post_send(id, remote_map, list_count, buffers, sizes,
@@ -1066 +1071 @@
   void *buffer, bmi_size_t expected_len, bmi_size_t *actual_len __unused,
   enum bmi_buffer_type buffer_flag __unused, bmi_msg_tag_t tag, void *user_ptr,
-  bmi_context_id context_id)
+  bmi_context_id context_id, PVFS_hint hints __unused)
 {
     return post_recv(id, remote_map, 0, &buffer, &expected_len,
@@ -1077 +1082 @@
   bmi_size_t tot_expected_len, bmi_size_t *tot_actual_len __unused,
   enum bmi_buffer_type buffer_flag __unused, bmi_msg_tag_t tag, void *user_ptr,
-  bmi_context_id context_id)
+  bmi_context_id context_id, PVFS_hint hints __unused)
 {
     return post_recv(id, remote_map, list_count, buffers, sizes,
@@ -2109 +2114 @@
 {
     .method_name = "bmi_ib",
+    .flags = 0,
     .initialize = BMI_ib_initialize,
     .finalize = BMI_ib_finalize,

branches/Orange-Branch/src/io/bmi/bmi_mx/README

@@ -68 +68 @@
     BMX_DEBUG                   Turn on gossip messages
     BMX_MEM_ACCT                Track memory usage
-    BMX_LOGGING                 Turn on MPE logging
     BMX_SERVER_RXS              Additional rxs for servers
     BMX_TIMEOUT                 Timeout for all MX messages
@@ -117 +116 @@
 BMI_mx_method_addr_lookup().
 
-BMX_LOGGING
-This is not generally recommended. It turns on support for MPE logging but
-it requires modifying the Makefile.in script and re-generating configure.
-Contact help <at> myri.com for assistance.
-
 BMX_SERVER_RXS
 The server will receive messages from unknown peers. This value determines how
@@ -156 +150 @@
 mx://hostname:ep_id
 
-Use the first option if mx_info lists hostname:board and use the second option if mx_info simply shows a hostname.
+Use the first option if mx_info lists hostname:board and use the second option
+if mx_info simply shows a hostname.
 
 ======================

branches/Orange-Branch/src/io/bmi/bmi_mx/mx.c

@@ -17 +17 @@
 struct bmx_data *bmi_mx = NULL; /* global state for bmi_mx */
 
+mx_status_t     BMX_NO_STATUS;
+
 #if BMX_MEM_ACCT
 uint64_t        mem_used = 0;   /* bytes used */
@@ -22 +24 @@
 #endif
 
-#if BMX_LOGGING
-int     send_start;
-int     send_finish;
-int     recv_start;
-int     recv_finish;
-int     sendunex_start;
-int     sendunex_finish;
-int     recvunex_start;
-int     recvunex_finish;
-#endif
-
 /* statics for event logging */
 static PINT_event_type bmi_mx_send_event_id;
@@ -48 +39 @@
 static void
 bmx_create_peername(void);
+
+/**** Completion function token handling ****************************/
+/* We should not hold any locks when calling mx_test[_any](),
+ * mx_wait_any() or mx_cancel(). We want to avoid races between them,
+ * however. So, before calling any completion function, the caller
+ * must hold this token.  These functions implement a token system (i.e.
+ * semaphore) that will wake up mx_wait_any() to reduce blocking times
+ * for the calling function.
+ */
+
+static void
+bmx_get_completion_token(void)
+{
+        int     done    = 0;
+
+        do {
+                gen_mutex_lock(&bmi_mx->bmx_completion_lock);
+                if (bmi_mx->bmx_refcount == 1) {
+                        bmi_mx->bmx_refcount--;
+                        done = 1;
+                        gen_mutex_unlock(&bmi_mx->bmx_completion_lock);
+                } else {
+                        assert(bmi_mx->bmx_refcount == 0);
+                        /* someone has the lock, wake the MX endpoint in
+                         * case they are blocking in mx_wait_any() */
+                        gen_mutex_unlock(&bmi_mx->bmx_completion_lock);
+                        mx_wakeup(bmi_mx->bmx_ep);
+                }
+        } while (!done);
+
+        return;
+}
+
+static void
+bmx_release_completion_token(void)
+{
+        gen_mutex_lock(&bmi_mx->bmx_completion_lock);
+        bmi_mx->bmx_refcount++;
+        assert(bmi_mx->bmx_refcount == 1);
+        gen_mutex_unlock(&bmi_mx->bmx_completion_lock);
+        return;
+}
 
 /**** TX/RX handling functions **************************************/
@@ -224 +257 @@
 }
 
-/* remove from peer's queued txs/rxs list */
-static void
-bmx_deq_ctx(struct bmx_ctx *ctx)
-{
-        struct bmx_peer *peer = ctx->mxc_peer;
-
-        BMX_ENTER;
-        if (!qlist_empty(&ctx->mxc_list)) {
-                gen_mutex_lock(&peer->mxp_lock);
-                qlist_del_init(&ctx->mxc_list);
-                gen_mutex_unlock(&peer->mxp_lock);
-        }
-        BMX_EXIT;
-        return;
-}
-
 /* add to peer's pending rxs list */
 static void
@@ -280 +297 @@
 }
 
-/* queue on unexpected rx or tx list */
+/* dequeue from unexpected rx list */
 static void
-bmx_q_unex_ctx(struct bmx_ctx *ctx)
-{
+bmx_deq_unex_rx(struct bmx_ctx **rxp)
+{
+        struct bmx_ctx  *rx     = NULL;
+        list_t          *list   = &bmi_mx->bmx_unex_rxs;
+
         BMX_ENTER;
-        if (ctx->mxc_type == BMX_REQ_RX) {
-                gen_mutex_lock(&bmi_mx->bmx_unex_rxs_lock);
-                qlist_add_tail(&ctx->mxc_list, &bmi_mx->bmx_unex_rxs);
-                gen_mutex_unlock(&bmi_mx->bmx_unex_rxs_lock);
-        } else {
-                gen_mutex_lock(&bmi_mx->bmx_unex_txs_lock);
-                qlist_add_tail(&ctx->mxc_list, &bmi_mx->bmx_unex_txs);
-                gen_mutex_unlock(&bmi_mx->bmx_unex_txs_lock);
-        }
+        gen_mutex_lock(&bmi_mx->bmx_unex_rxs_lock);
+        if (!qlist_empty(list)) {
+                rx = qlist_entry(list->next, struct bmx_ctx, mxc_list);
+                qlist_del_init(&rx->mxc_list);
+        }
+        gen_mutex_unlock(&bmi_mx->bmx_unex_rxs_lock);
+        *rxp = rx;
+
         BMX_EXIT;
         return;
 }
 
-/* add to the global canceled list */
+/* add to the completion queue for the appropriate context */
 static void
-bmx_q_canceled_ctx(struct bmx_ctx *ctx, bmi_error_code_t error)
-{
+bmx_q_completed(struct bmx_ctx *ctx, enum bmx_ctx_state state,
+                mx_status_t status, bmi_error_code_t error)
+{
+        int             id      = 0;
+        gen_mutex_t     *lock   = NULL;
+        list_t          *list   = NULL;
+
         BMX_ENTER;
-        ctx->mxc_state = BMX_CTX_CANCELED;
-        if (error < 0)
-                ctx->mxc_mxstat.code = error;
-        else
-                ctx->mxc_mxstat.code = -error;
-        gen_mutex_lock(&bmi_mx->bmx_canceled_lock);
-        qlist_add_tail(&ctx->mxc_list, &bmi_mx->bmx_canceled);
-        gen_mutex_unlock(&bmi_mx->bmx_canceled_lock);
+
+        ctx->mxc_state = state;
+        ctx->mxc_mxstat = status;
+        ctx->mxc_error = error < 0 ? error : -error;
+
+        if (ctx->mxc_type == BMX_REQ_RX &&
+            ctx->mxc_msg_type == BMX_MSG_UNEXPECTED) {
+                list = &bmi_mx->bmx_unex_rxs;
+                lock = &bmi_mx->bmx_unex_rxs_lock;
+        } else {
+                id = (int) ctx->mxc_mop->context_id;
+                lock = &bmi_mx->bmx_done_q_lock[id];
+                list = &bmi_mx->bmx_done_q[id];
+        }
+
+
+        gen_mutex_lock(lock);
+        qlist_add_tail(&ctx->mxc_list, list);
+        gen_mutex_unlock(lock);
+        BMX_EXIT;
+        return;
+}
+
+static void
+bmx_deq_completed(struct bmx_ctx **ctxp, bmi_context_id context_id)
+{
+        int             id      = (int) context_id;
+        list_t          *list   = &bmi_mx->bmx_done_q[id];
+        gen_mutex_t     *lock   = &bmi_mx->bmx_done_q_lock[id];
+        struct bmx_ctx  *ctx    = NULL;
+
+        BMX_ENTER;
+
+        gen_mutex_lock(lock);
+        if (!qlist_empty(list)) {
+                ctx = qlist_entry(list->next, struct bmx_ctx, mxc_list);
+                qlist_del_init(&ctx->mxc_list);
+        }
+        gen_mutex_unlock(lock);
+        *ctxp = ctx;
+
         BMX_EXIT;
         return;
@@ -347 +404 @@
 
 static void
-bmx_put_idle_rx(struct bmx_ctx *rx)
-{
-        if (rx == NULL) {
-                debug(BMX_DB_WARN, "put_idle_rx() called with NULL");
+bmx_put_idle_ctx(struct bmx_ctx *ctx)
+{
+        list_t          *list   = &bmi_mx->bmx_idle_txs;
+        gen_mutex_t     *lock   = &bmi_mx->bmx_idle_txs_lock;
+
+        if (ctx == NULL) {
+                debug(BMX_DB_WARN, "put_idle_ctx() called with NULL");
                 return;
         }
-        if (rx->mxc_type != BMX_REQ_RX) {
-                debug(BMX_DB_WARN, "put_idle_rx() called with a TX");
-                return;
-        }
-        if (rx->mxc_get != rx->mxc_put + 1) {
-                debug(BMX_DB_ERR, "put_idle_rx() get (%llu) != put (%llu) + 1",
-                         (unsigned long long) rx->mxc_get,
-                         (unsigned long long) rx->mxc_put);
+        ctx->mxc_put++;
+        if (ctx->mxc_get != ctx->mxc_put) {
+                debug(BMX_DB_ERR, "put_idle_ctx() get (%llu) != put (%llu)",
+                         (unsigned long long) ctx->mxc_get,
+                         (unsigned long long) ctx->mxc_put);
                 exit(1);
         }
-        bmx_ctx_init(rx);
-        rx->mxc_put++;
-        gen_mutex_lock(&bmi_mx->bmx_idle_rxs_lock);
-        qlist_add(&rx->mxc_list, &bmi_mx->bmx_idle_rxs);
-        gen_mutex_unlock(&bmi_mx->bmx_idle_rxs_lock);
+        bmx_ctx_init(ctx);
+
+        if (ctx->mxc_type == BMX_REQ_RX) {
+                list   = &bmi_mx->bmx_idle_rxs;
+                lock   = &bmi_mx->bmx_idle_rxs_lock;
+        }
+
+        gen_mutex_lock(lock);
+        qlist_add(&ctx->mxc_list, list);
+        gen_mutex_unlock(lock);
         return;
 }
@@ -416 +478 @@
 
         return tx;
-}
-
-static void
-bmx_put_idle_tx(struct bmx_ctx *tx)
-{
-        if (tx == NULL) {
-                debug(BMX_DB_WARN, "put_idle_tx() called with NULL");
-                return;
-        }
-        if (tx->mxc_type != BMX_REQ_TX) {
-                debug(BMX_DB_WARN, "put_idle_tx() called with a TX");
-                return;
-        }
-        if (tx->mxc_get != tx->mxc_put + 1) {
-                debug(BMX_DB_ERR, "put_idle_tx() get (%llu) != put (%llu) + 1",
-                         (unsigned long long) tx->mxc_get,
-                         (unsigned long long) tx->mxc_put);
-                exit(1);
-        }
-        bmx_ctx_init(tx);
-        tx->mxc_put++;
-        gen_mutex_lock(&bmi_mx->bmx_idle_txs_lock);
-        qlist_add(&tx->mxc_list, &bmi_mx->bmx_idle_txs);
-        gen_mutex_unlock(&bmi_mx->bmx_idle_txs_lock);
-        return;
 }
 
@@ -759 +796 @@
                         return ret;
                 }
-                bmx_put_idle_rx(rx);
+                bmx_put_idle_ctx(rx);
         }
 
@@ -810 +847 @@
 bmx_globals_init(int method_id)
 {
+        int     i       = 0;
+
 #if BMX_MEM_ACCT
         mem_used = 0;
@@ -840 +879 @@
         gen_mutex_init(&bmi_mx->bmx_idle_rxs_lock);
 
-        INIT_QLIST_HEAD(&bmi_mx->bmx_canceled);
-        gen_mutex_init(&bmi_mx->bmx_canceled_lock);
-
-        INIT_QLIST_HEAD(&bmi_mx->bmx_unex_txs);
-        gen_mutex_init(&bmi_mx->bmx_unex_txs_lock);
+        gen_mutex_init(&bmi_mx->bmx_completion_lock);
+        /* set to 1 to allow testing to start */
+        bmi_mx->bmx_refcount = 1;
+
+        for (i = 0; i < BMI_MAX_CONTEXTS; i++) {
+                INIT_QLIST_HEAD(&bmi_mx->bmx_done_q[i]);
+                gen_mutex_init(&bmi_mx->bmx_done_q_lock[i]);
+        }
+
         INIT_QLIST_HEAD(&bmi_mx->bmx_unex_rxs);
         gen_mutex_init(&bmi_mx->bmx_unex_rxs_lock);
@@ -883 +926 @@
         param.val.context_id.shift = BMX_MSG_SHIFT;
 
-        if (board == -1) board = 0;
         mxret = mx_open_endpoint(board, ep_id, BMX_MAGIC,
                                  &param, 1, ep);
@@ -922 +964 @@
 
         BMX_ENTER;
-
-#if BMX_LOGGING
-        MPE_Init_log();
-#define BMX_LOG_STATE 1
-#if BMX_LOG_STATE
-        send_start              = MPE_Log_get_event_number();
-        send_finish             = MPE_Log_get_event_number();
-        recv_start              = MPE_Log_get_event_number();
-        recv_finish             = MPE_Log_get_event_number();
-        sendunex_start          = MPE_Log_get_event_number();
-        sendunex_finish         = MPE_Log_get_event_number();
-        recvunex_start          = MPE_Log_get_event_number();
-        recvunex_finish         = MPE_Log_get_event_number();
-        MPE_Describe_state(send_start, send_finish, "Send", "red");
-        MPE_Describe_state(recv_start, recv_finish, "Recv", "blue");
-        MPE_Describe_state(sendunex_start, sendunex_finish, "SendUnex", "orange");
-        MPE_Describe_state(recvunex_start, recvunex_finish, "RecvUnex", "green");
-#else
-        MPE_Log_get_solo_eventID(&send_start);
-        MPE_Log_get_solo_eventID(&send_finish);
-        MPE_Log_get_solo_eventID(&recv_start);
-        MPE_Log_get_solo_eventID(&recv_finish);
-        MPE_Log_get_solo_eventID(&sendunex_start);
-        MPE_Log_get_solo_eventID(&sendunex_finish);
-        MPE_Log_get_solo_eventID(&recvunex_start);
-        MPE_Log_get_solo_eventID(&recvunex_finish);
-        MPE_Describe_info_event(send_start, "Send_start", "red1", "tag:%d");
-        MPE_Describe_info_event(send_finish, "Send_finish", "red3", "tag:%d");
-        MPE_Describe_info_event(recv_start, "Recv_start", "blue1", "tag:%d");
-        MPE_Describe_info_event(recv_finish, "Recv_finish", "blue3", "tag:%d");
-        MPE_Describe_info_event(sendunex_start, "SendUnex_start", "orange1", "tag:%d");
-        MPE_Describe_info_event(sendunex_finish, "SendUnex_finish", "orange3", "tag:%d");
-        MPE_Describe_info_event(recvunex_start, "RecvUnex_start", "green1", "tag:%d");
-        MPE_Describe_info_event(recvunex_finish, "RecvUnex_finish", "green3", "tag:%d");
-#endif /* state or event */
-#endif /* BMX_LOGGING */
 
          /* check params */
@@ -1024 +1030 @@
                         ret = bmx_ctx_alloc(&rx, BMX_REQ_RX);
                         if (ret == 0) {
-                                bmx_put_idle_rx(rx);
+                                bmx_put_idle_ctx(rx);
                         }
                 }
@@ -1137 +1143 @@
         debug(BMX_DB_MEM, "memory leaked at shutdown %lld", llu(mem_used));
 #endif
-
-#if BMX_LOGGING
-        MPE_Finish_log("/tmp/bmi_mx.log");
-#endif
         BMX_EXIT;
-
         return 0;
 }
@@ -1180 +1181 @@
                 tx = qlist_entry(queued_txs->next, struct bmx_ctx, mxc_list);
                 qlist_del_init(&tx->mxc_list);
-                bmx_q_canceled_ctx(tx, err);
+                bmx_q_completed(tx, BMX_CTX_CANCELED, BMX_NO_STATUS, err);
         }
         /* cancel queued rxs */
@@ -1187 +1188 @@
                 rx = qlist_entry(queued_rxs->next, struct bmx_ctx, mxc_list);
                 qlist_del_init(&rx->mxc_list);
-                bmx_q_canceled_ctx(rx, err);
+                bmx_q_completed(rx, BMX_CTX_CANCELED, BMX_NO_STATUS, err);
         }
         /* try to cancel pending rxs */
@@ -1195 +1196 @@
                 if (result) {
                         qlist_del_init(&rx->mxc_list);
-                        bmx_q_canceled_ctx(rx, err);
+                        bmx_q_completed(rx, BMX_CTX_CANCELED, BMX_NO_STATUS, err);
                 }
         }
@@ -1424 +1425 @@
         uint64_t        type    = (uint64_t) ctx->mxc_msg_type;
         uint64_t        id      = 0ULL;
-        uint64_t        tag     = (uint64_t) ctx->mxc_tag;
+        uint64_t        tag     = (uint64_t) ((uint32_t) ctx->mxc_tag);
 
         if (ctx->mxc_msg_type == BMX_MSG_CONN_REQ || 
@@ -1517 +1518 @@
                         ret = -BMI_ENOMEM;
                         bmx_deq_pending_ctx(tx);        /* uses peer lock */
-                        bmx_q_canceled_ctx(tx, BMI_ENOMEM);
+                        bmx_q_completed(tx, BMX_CTX_CANCELED, BMX_NO_STATUS, BMI_ENOMEM);
                 }
         } else { /* peer is not ready */
@@ -1572 +1573 @@
         PINT_event_id            eid    = 0;
 
-#if BMX_LOGGING
-        if (!is_unexpected) {
-                MPE_Log_event(send_start, (int) tag, NULL);
-        } else {
-                MPE_Log_event(sendunex_start, (int) tag, NULL);
-        }
-#endif
-
         PINT_EVENT_START(
             bmi_mx_send_event_id, bmi_mx_pid, NULL, &eid,
@@ -1621 +1614 @@
                 BMX_MALLOC(segs, (numbufs * sizeof(*segs)));
                 if (segs == NULL) {
-                        bmx_put_idle_tx(tx);
+                        bmx_put_idle_ctx(tx);
                         bmx_peer_decref(peer);
                         ret = -BMI_ENOMEM;
@@ -1642 +1635 @@
 
         if (is_unexpected && tx->mxc_nob > (long long) BMX_UNEXPECTED_SIZE) {
-                bmx_put_idle_tx(tx);
+                bmx_put_idle_ctx(tx);
                 bmx_peer_decref(peer);
                 ret = -BMI_EINVAL;
@@ -1658 +1651 @@
         BMX_MALLOC(mop, sizeof(*mop));
         if (mop == NULL) {
-                bmx_put_idle_tx(tx);
+                bmx_put_idle_ctx(tx);
                 bmx_peer_decref(peer);
                 ret = -BMI_ENOMEM;
@@ -1673 +1666 @@
         tx->mxc_mop = mop;
 
+        assert(context_id == mop->context_id);
+        assert(context_id == tx->mxc_mop->context_id);
+
         bmx_create_match(tx);
 
-        debug(BMX_DB_CTX, "%s tag= %d length= %d %s op_id= %llu", __func__, tag,
-                        (int) total_size, is_unexpected ? "UNEXPECTED" : "EXPECTED",
-                        llu(mop->op_id));
+        debug(BMX_DB_CTX, "%s tag= %d length= %d %s op_id= %llu context_id= %lld",
+                        __func__, tag, (int) total_size,
+                        is_unexpected ? "UNEXPECTED" : "EXPECTED",
+                        llu(mop->op_id), lld(context_id));
 
         ret = bmx_post_tx(tx);
@@ -1792 +1789 @@
                         ret = -BMI_ENOMEM;
                         bmx_deq_pending_ctx(rx);        /* uses peer lock */
-                        bmx_q_canceled_ctx(rx, BMI_ENOMEM);
+                        bmx_q_completed(rx, BMX_CTX_CANCELED, BMX_NO_STATUS, BMI_ENOMEM);
                 }
         } else { /* peer is not ready */
@@ -1817 +1814 @@
         struct bmx_peer         *peer   = NULL;
         PINT_event_id            eid    = 0;
-
-#if BMX_LOGGING
-        MPE_Log_event(recv_start, (int) tag, NULL);
-#endif
 
         PINT_EVENT_START(
@@ -1865 +1858 @@
                 BMX_MALLOC(segs, (numbufs * sizeof(*segs)));
                 if (segs == NULL) {
-                        bmx_put_idle_rx(rx);
+                        bmx_put_idle_ctx(rx);
                         bmx_peer_decref(peer);
                         ret = -BMI_ENOMEM;
@@ -1887 +1880 @@
         BMX_MALLOC(mop, sizeof(*mop));
         if (mop == NULL) {
-                bmx_put_idle_rx(rx);
+                bmx_put_idle_ctx(rx);
                 bmx_peer_decref(peer);
                 ret = -BMI_ENOMEM;
@@ -2026 +2019 @@
         }
 
-#if BMX_LOGGING
-        MPE_Log_event(recvunex_start, (int) tag, NULL);
-#endif
-
         rx = bmx_get_idle_rx();
         if (rx != NULL) {
@@ -2062 +2051 @@
                                         "unexpected recv with tag %d length %d",
                                         mx_strerror(mxret), tag, length);
-                        bmx_put_idle_rx(rx);
+                        bmx_put_idle_ctx(rx);
                         ret = -1;
                 }
@@ -2121 +2110 @@
                                                 "unexpected recv with %s", 
                                                 mx_strerror(mxret));
-                                bmx_put_idle_rx(rx);
+                                bmx_put_idle_ctx(rx);
                                 ret = MX_RECV_FINISHED;
                         }
@@ -2341 +2330 @@
                                 /* drop ref taken before mx_iconnect() */
                                 bmx_peer_decref(tx->mxc_peer);
-                                bmx_put_idle_tx(tx);
+                                bmx_put_idle_ctx(tx);
                                 continue;
                         } else if (status.code != MX_STATUS_SUCCESS) {
                                 bmx_peer_decref(rx->mxc_peer);
-                                bmx_put_idle_rx(rx);
+                                bmx_put_idle_ctx(rx);
                                 continue;
                         }
@@ -2356 +2345 @@
                                                 BMX_VERSION, version);
                                 bmx_peer_decref(rx->mxc_peer);
-                                bmx_put_idle_rx(rx);
+                                bmx_put_idle_ctx(rx);
                                 continue;
                         }
@@ -2362 +2351 @@
                                 debug(BMX_DB_WARN, "received CONN_REQ on a client.");
                                 bmx_peer_decref(rx->mxc_peer);
-                                bmx_put_idle_rx(rx);
+                                bmx_put_idle_ctx(rx);
                                 continue;
                         }
@@ -2390 +2379 @@
                                                         (char *) rx->mxc_buffer);
                                         bmx_peer_decref(rx->mxc_peer);
-                                        bmx_put_idle_rx(rx);
+                                        bmx_put_idle_ctx(rx);
                                         continue;
                                 }
@@ -2399 +2388 @@
                                                         "method addr for %s", peername);
                                         bmx_peer_decref(rx->mxc_peer);
-                                        bmx_put_idle_rx(rx);
+                                        bmx_put_idle_ctx(rx);
                                         continue;
                                 }
@@ -2409 +2398 @@
                                                         "peer for %s", peername);
                                         bmx_peer_decref(rx->mxc_peer);
-                                        bmx_put_idle_rx(rx);
+                                        bmx_put_idle_ctx(rx);
                                         continue;
                                 }
@@ -2436 +2425 @@
                         mx_iconnect(bmi_mx->bmx_ep, peer->mxp_nic_id, mxmap->mxm_ep_id,
                                     BMX_MAGIC, ack, peer, &request);
-                        bmx_put_idle_rx(rx);
+                        bmx_put_idle_ctx(rx);
                 }
         } while (result);
@@ -2551 +2540 @@
                                         llu(tx->mxc_match), mx_strstatus(status.code));
                         bmx_peer_decref(tx->mxc_peer);
-                        bmx_put_idle_tx(tx);
+                        bmx_put_idle_ctx(tx);
                 }
         } while (result);
@@ -2578 +2567 @@
 }
 
+static void
+bmx_complete_ctx(struct bmx_ctx *ctx, bmi_op_id_t *outid, bmi_error_code_t *err,
+                 bmi_size_t *size, void **user_ptr)
+{
+        struct bmx_peer *peer   = ctx->mxc_peer;
+
+        *outid = ctx->mxc_mop->op_id;
+        *err = ctx->mxc_error;
+        *size = ctx->mxc_mxstat.xfer_length;
+        if (user_ptr)
+                *user_ptr = ctx->mxc_mop->user_ptr;
+        PINT_EVENT_END(
+            (ctx->mxc_type == BMX_REQ_TX ?
+             bmi_mx_send_event_id : bmi_mx_recv_event_id),
+            bmi_mx_pid, NULL, ctx->mxc_mop->event_id,
+            *outid, *size);
+
+        id_gen_fast_unregister(ctx->mxc_mop->op_id);
+        BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
+        bmx_put_idle_ctx(ctx);
+        bmx_peer_decref(peer); /* drop the ref taken in [send|recv]_common */
+
+        return;
+}
+
 static int
 BMI_mx_test(bmi_op_id_t id, int *outcount, bmi_error_code_t *err,
             bmi_size_t *size, void **user_ptr, int max_idle_time __unused,
-            bmi_context_id context_id __unused)
+            bmi_context_id context_id)
 {
         uint32_t         result = 0;
@@ -2592 +2606 @@
         bmx_connection_handlers();
 
+        bmx_get_completion_token();
+
         mop = id_gen_fast_lookup(id);
         ctx = mop->method_data;
         peer = ctx->mxc_peer;
 
+        assert(context_id == mop->context_id);
+        if (ctx->mxc_type == BMX_REQ_RX)
+                assert(ctx->mxc_msg_type != BMX_MSG_UNEXPECTED);
+        assert(context_id == ctx->mxc_mop->context_id);
+
         switch (ctx->mxc_state) {
+        case BMX_CTX_COMPLETED:
         case BMX_CTX_CANCELED:
-                /* we are racing with testcontext */
-                gen_mutex_lock(&bmi_mx->bmx_canceled_lock);
-                if (ctx->mxc_state != BMX_CTX_CANCELED) {
-                        gen_mutex_unlock(&bmi_mx->bmx_canceled_lock);
-                        return 0;
-                }
+                gen_mutex_lock(&bmi_mx->bmx_done_q_lock[(int) context_id]);
                 qlist_del_init(&ctx->mxc_list);
-                gen_mutex_unlock(&bmi_mx->bmx_canceled_lock);
+                gen_mutex_unlock(&bmi_mx->bmx_done_q_lock[(int) context_id]);
+                bmx_complete_ctx(ctx, &id, err, size, user_ptr);
                 *outcount = 1;
-                *err = ctx->mxc_mxstat.code;
-                if (ctx->mxc_mop) {
-                        if (user_ptr) {
-                                *user_ptr = ctx->mxc_mop->user_ptr;
-                        }
-                        id_gen_fast_unregister(ctx->mxc_mop->op_id);
-                        BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
-                }
-                bmx_peer_decref(peer);
-                if (ctx->mxc_type == BMX_REQ_TX) {
-                        bmx_put_idle_tx(ctx);
-                } else {
-                        bmx_put_idle_rx(ctx);
-                }
                 break;
         case BMX_CTX_PENDING:
-                /* racing with mx_test_any() in textcontext? */
                 mx_test(bmi_mx->bmx_ep, &ctx->mxc_mxreq, &ctx->mxc_mxstat, &result);
                 if (result) {
-                        PINT_EVENT_END(
-                            (ctx->mxc_type == BMX_REQ_TX ?
-                             bmi_mx_send_event_id : bmi_mx_recv_event_id),
-                            bmi_mx_pid, NULL, ctx->mxc_mop->event_id,
-                            ctx->mxc_mop->op_id, *size);
-
+                        bmx_deq_pending_ctx(ctx);
+                        bmx_complete_ctx(ctx, &id, err, size, user_ptr);
                         *outcount = 1;
-                        if (ctx->mxc_mxstat.code == MX_STATUS_SUCCESS) {
-                                *err = 0;
-                                *size = ctx->mxc_mxstat.xfer_length;
-                        } else {
-                                *err = bmx_mx_to_bmi_errno(ctx->mxc_mxstat.code);
-                        }
-                        if (ctx->mxc_mop) {
-                                if (user_ptr) {
-                                        *user_ptr = ctx->mxc_mop->user_ptr;
-                                }
-                                id_gen_fast_unregister(ctx->mxc_mop->op_id);
-                                BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
-                        }
-                        bmx_deq_pending_ctx(ctx);
-                        if (ctx->mxc_type == BMX_REQ_TX) {
-                                bmx_put_idle_tx(ctx);
-                        } else {
-                                bmx_put_idle_rx(ctx);
-                        }
-                        bmx_peer_decref(peer);
                 }
                 break;
@@ -2659 +2638 @@
                         ctx->mxc_type == BMX_REQ_TX ? "TX" : "RX", ctx->mxc_state);
         }
+        bmx_release_completion_token();
         BMX_EXIT;
 
@@ -2677 +2657 @@
         struct bmx_ctx  *ctx            = NULL;
         struct bmx_peer *peer           = NULL;
-        list_t          *canceled       = &bmi_mx->bmx_canceled;
         int             wait            = 0;
         static int      count           = 0;
@@ -2689 +2668 @@
         bmx_connection_handlers();
 
-        /* always return canceled messages first */
-        while (completed < incount && !qlist_empty(canceled)) {
-                gen_mutex_lock(&bmi_mx->bmx_canceled_lock);
-                ctx = qlist_entry(canceled->next, struct bmx_ctx, mxc_list);
-                qlist_del_init(&ctx->mxc_list);
-                /* change state in case test is trying to reap it as well */
-                ctx->mxc_state = BMX_CTX_COMPLETED;
-                gen_mutex_unlock(&bmi_mx->bmx_canceled_lock);
-                peer = ctx->mxc_peer;
-                outids[completed] = ctx->mxc_mop->op_id;
-                errs[completed] = ctx->mxc_mxstat.code;
-                if (user_ptrs)
-                        user_ptrs[completed] = ctx->mxc_mop->user_ptr;
-                id_gen_fast_unregister(ctx->mxc_mop->op_id);
-                BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
-                completed++;
-                if (ctx->mxc_type == BMX_REQ_TX) {
-                        bmx_put_idle_tx(ctx);
-                } else {
-                        bmx_put_idle_rx(ctx);
-                }
-                bmx_peer_decref(peer); /* drop the ref taken in [send|recv]_common */
-                if (completed > 0) {
-                        debug(BMX_DB_CTX, "%s found %d canceled messages",
-                              __func__, completed);
-                }
-        }
-
-        /* return completed messages
+        bmx_get_completion_token();
+
+        /* always return queued, completed messages first */
+        do {
+                bmx_deq_completed(&ctx, context_id);
+                if (ctx) {
+                        bmx_complete_ctx(ctx, &outids[completed], &errs[completed],
+                                         &sizes[completed], &user_ptrs[completed]);
+                        completed++;
+                }
+        } while (completed < incount && ctx != NULL);
+
+        if (completed > 0)
+                debug(BMX_DB_CTX, "%s found %d completed messages", __func__, completed);
+
+        /* try to complete expected messages
          * we will always try (incount - completed) times even
          *     if some iterations have no result */
@@ -2736 +2702 @@
                         wait = 2;
                 }
+
                 if (result) {
                         ctx = (struct bmx_ctx *) status.context;
                         bmx_deq_pending_ctx(ctx);
+                        if (ctx->mxc_mop->context_id != context_id) {
+                                bmx_q_completed(ctx, BMX_CTX_COMPLETED, status,
+                                                bmx_mx_to_bmi_errno(status.code));
+                                continue;
+                        }
+                        ctx->mxc_mxstat = status;
                         peer = ctx->mxc_peer;
                         debug(BMX_DB_CTX, "%s completing expected %s with match 0x%llx "
-                                        "for %s with op_id %llu length %d %s", __func__, 
-                                        ctx->mxc_type == BMX_REQ_TX ? "TX" : "RX",
+                                        "for %s with op_id %llu length %d %s "
+                                        "context_id= %d mop->context_id= %d",
+                                        __func__, ctx->mxc_type == BMX_REQ_TX ? "TX" : "RX",
                                         llu(ctx->mxc_match), peer->mxp_mxmap->mxm_peername,
                                         llu(ctx->mxc_mop->op_id), status.xfer_length,
-                                        mx_strstatus(status.code));
-
-                        outids[completed] = ctx->mxc_mop->op_id;
-                        if (status.code == MX_SUCCESS) {
-                                errs[completed] = 0;
-                                sizes[completed] = status.xfer_length;
-                        } else {
-                                errs[completed] = bmx_mx_to_bmi_errno(status.code);
-                        }
-                        if (user_ptrs)
-                                user_ptrs[completed] = ctx->mxc_mop->user_ptr;
-
-                        PINT_EVENT_END(
-                            (ctx->mxc_type == BMX_REQ_TX ?
-                             bmi_mx_send_event_id : bmi_mx_recv_event_id),
-                            bmi_mx_pid, NULL, ctx->mxc_mop->event_id,
-                            ctx->mxc_mop->op_id, status.xfer_length);
-
-                        id_gen_fast_unregister(ctx->mxc_mop->op_id);
-                        BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
+                                        mx_strstatus(status.code), (int) context_id,
+                                        (int) ctx->mxc_mop->context_id);
+
+                        bmx_complete_ctx(ctx, &outids[completed], &errs[completed],
+                                         &sizes[completed], &user_ptrs[completed]);
                         completed++;
-#if BMX_LOGGING
-                        if (ctx->mxc_type == BMX_REQ_TX) {
-                                MPE_Log_event(send_finish, (int) ctx->mxc_tag, NULL);
-                        } else {
-                                MPE_Log_event(recv_finish, (int) ctx->mxc_tag, NULL);
-                        }
-#endif
-                        if (ctx->mxc_type == BMX_REQ_TX) {
-                                bmx_put_idle_tx(ctx);
-                        } else {
-                                bmx_put_idle_rx(ctx);
-                        }
-                        bmx_peer_decref(peer); /* drop the ref taken in [send|recv]_common */
-                }
-                if (completed - old > 0) {
-                        debug(BMX_DB_CTX, "%s found %d expected messages",
-                              __func__, completed - old);
-                }
-        }
-
-        /* check for completed unexpected sends */
+                }
+        }
+
+        if (completed - old > 0)
+                debug(BMX_DB_CTX, "%s found %d expected messages", __func__, completed - old);
+
+        /* try to complete unexpected sends */
 
         match = (uint64_t) BMX_MSG_UNEXPECTED << BMX_MSG_SHIFT;
@@ -2795 +2740 @@
                 uint32_t        result          = 0;
                 mx_status_t     status;
-                list_t          *unex_txs       = &bmi_mx->bmx_unex_txs;
                 int             again           = 1;
 
                 ctx = NULL;
-
-                gen_mutex_lock(&bmi_mx->bmx_unex_txs_lock);
-                if (!qlist_empty(unex_txs)) {
-                        ctx = qlist_entry(unex_txs->next, struct bmx_ctx, mxc_list);
-                        peer = ctx->mxc_peer;
-                        qlist_del_init(&ctx->mxc_list);
-                        result = 1;
-                }
-                gen_mutex_unlock(&bmi_mx->bmx_unex_txs_lock);
 
                 while (!ctx && again) {
@@ -2816 +2751 @@
                                 bmx_deq_pending_ctx(ctx);
                                 peer = ctx->mxc_peer;
-                                if (ctx->mxc_type == BMX_REQ_RX) {
-                                        /* queue until testunexpected is called */
-                                        bmx_q_unex_ctx(ctx);
+                                if (ctx->mxc_type == BMX_REQ_RX ||
+                                    ctx->mxc_mop->context_id != context_id) {
+                                        /* queue until testunexpected or queue
+                                         * until testcontext for the correct context */
+                                        bmx_q_completed(ctx, BMX_CTX_COMPLETED, status,
+                                                        bmx_mx_to_bmi_errno(status.code));
                                         result = 0;
                                         again = 1;
@@ -2834 +2772 @@
                                         llu(ctx->mxc_mop->op_id));
 
-                        outids[completed] = ctx->mxc_mop->op_id;
-                        if (status.code == MX_SUCCESS) {
-                                errs[completed] = 0;
-                                sizes[completed] = status.xfer_length;
-                        } else {
-                                errs[completed] = bmx_mx_to_bmi_errno(status.code);
+                        ctx->mxc_mxstat = status;
+                        bmx_complete_ctx(ctx, &outids[completed], &errs[completed],
+                                         &sizes[completed], &user_ptrs[completed]);
+
+                        if (status.code != MX_SUCCESS) {
                                 debug(BMX_DB_CTX, "%s unexpected send completed with "
                                       "error %s", __func__, mx_strstatus(status.code));
                                 bmx_peer_disconnect(peer, 0, BMI_ENETRESET);
                         }
-                        if (user_ptrs)
-                                user_ptrs[completed] = ctx->mxc_mop->user_ptr;
-                        PINT_EVENT_END(
-                            (ctx->mxc_type == BMX_REQ_TX ?
-                             bmi_mx_send_event_id : bmi_mx_recv_event_id),
-                            bmi_mx_pid, NULL, ctx->mxc_mop->event_id,
-                            ctx->mxc_mop->op_id, status.xfer_length);
-
-                        id_gen_fast_unregister(ctx->mxc_mop->op_id);
-                        BMX_FREE(ctx->mxc_mop, sizeof(*ctx->mxc_mop));
                         completed++;
-#if BMX_LOGGING
-                        MPE_Log_event(sendunex_finish, (int) ctx->mxc_tag, NULL);
-#endif
-
-                        bmx_put_idle_tx(ctx);
-                        bmx_peer_decref(peer); /* drop the ref taken in [send|recv]_common */
-                }
-        }
+                }
+        }
+        bmx_release_completion_token();
+
         if (completed - old > 0) {
                 debug(BMX_DB_CTX, "%s found %d unexpected tx messages", 
@@ -2888 +2811 @@
         struct bmx_ctx  *rx             = NULL;
         struct bmx_peer *peer           = NULL;
-        list_t          *unex_rxs       = &bmi_mx->bmx_unex_rxs;
         int             again           = 1;
 
@@ -2897 +2819 @@
 
         bmx_connection_handlers();
+
+        bmx_get_completion_token();
 
         /* if the unexpected handler cannot get a rx, it does not post a receive.
@@ -2903 +2827 @@
         if (result) {
                 int     ret     = 0;
-                ret = bmx_post_unexpected_recv(status.source, 0, 0, 0, status.match_info, status.xfer_length);
+                ret = bmx_post_unexpected_recv(status.source, 0, 0, 0,
+                                               status.match_info,
+                                               status.xfer_length);
                 if (ret != 0) {
                         debug(BMX_DB_CTX, "%s mx_iprobe() found rx with match 0x%llx "
@@ -2914 +2840 @@
         *outcount = 0;
 
-        gen_mutex_lock(&bmi_mx->bmx_unex_rxs_lock);
-        if (!qlist_empty(unex_rxs)) {
-                rx = qlist_entry(unex_rxs->next, struct bmx_ctx, mxc_list);
+        bmx_deq_unex_rx(&rx);
+        if (rx) {
+                result = 1;
+                status = rx->mxc_mxstat;
                 peer = rx->mxc_peer;
-                qlist_del_init(&rx->mxc_list);
-                result = 1;
-        }
-        gen_mutex_unlock(&bmi_mx->bmx_unex_rxs_lock);
+        }
 
         while (!rx && again) {
@@ -2931 +2855 @@
                         peer = rx->mxc_peer;
                         if (rx->mxc_type == BMX_REQ_TX) {
-                                bmx_q_unex_ctx(rx);
+                                bmx_q_completed(rx, BMX_CTX_COMPLETED, status,
+                                                bmx_mx_to_bmi_errno(status.code));
                                 result = 0;
                                 again = 1;
@@ -2953 +2878 @@
                 ui->tag = rx->mxc_tag;
 
-#if BMX_LOGGING
-                MPE_Log_event(recvunex_finish, (int) rx->mxc_tag, NULL);
-#endif
-                bmx_put_idle_rx(rx);
+                bmx_put_idle_ctx(rx);
                 bmx_peer_decref(peer); /* drop the ref taken in unexpected_recv() */
                 *outcount = 1;
         }
+        bmx_release_completion_token();
+
         if (print)
                 BMX_EXIT;
@@ -3014 +2938 @@
 
                 ret = bmx_open_endpoint(&bmi_mx->bmx_ep,
-                                        bmi_mx->bmx_board,
+                                        MX_ANY_NIC,
                                         MX_ANY_ENDPOINT);
                 if (ret != 0) {
@@ -3026 +2950 @@
                 mx_decompose_endpoint_addr2(epa, &nic_id, &bmi_mx->bmx_ep_id,
                                             &bmi_mx->bmx_sid);
+                /* get our board number */
+                mx_nic_id_to_board_number(nic_id, &bmi_mx->bmx_board);
                 /* get our hostname */
                 mx_nic_id_to_hostname(nic_id, host);
@@ -3141 +3067 @@
 /* NOTE There may be a race between this and BMI_mx_testcontext(). */
 static int
-BMI_mx_cancel(bmi_op_id_t id, bmi_context_id context_id __unused)
+BMI_mx_cancel(bmi_op_id_t id, bmi_context_id context_id)
 {
         struct method_op        *mop;
@@ -3150 +3076 @@
         BMX_ENTER;
 
+        bmx_get_completion_token();
+
         mop = id_gen_fast_lookup(id);
         ctx = mop->method_data;
         peer = ctx->mxc_peer;
 
+        assert(context_id == ctx->mxc_mop->context_id);
+
         debug(BMX_DB_CTX, "%s %s op_id %llu mxc_state %d peer state %d", __func__, 
                         ctx->mxc_type == BMX_REQ_TX ? "TX" : "RX", 
                         llu(ctx->mxc_mop->op_id), ctx->mxc_state, peer->mxp_state);
+
+        /* avoid race with connection setup */
+        gen_mutex_lock(&peer->mxp_lock);
+
         switch (ctx->mxc_state) {
         case BMX_CTX_QUEUED:
-                /* we are racing with the connection setup */
-                bmx_deq_ctx(ctx);
-                bmx_q_canceled_ctx(ctx,  BMI_ECANCEL);
+                qlist_del_init(&ctx->mxc_list);
+                gen_mutex_unlock(&peer->mxp_lock);
+                bmx_q_completed(ctx, BMX_CTX_CANCELED, BMX_NO_STATUS, BMI_ECANCEL);
                 break;
         case BMX_CTX_PENDING:
+                gen_mutex_unlock(&peer->mxp_lock);
                 if (ctx->mxc_type == BMX_REQ_TX) {
                         /* see if it completed first */
@@ -3170 +3105 @@
                                 debug(BMX_DB_CTX, "%s completed TX op_id %llu "
                                       "mxc_state %d peer state %d status.code %s",
-                                      __func__, llu(ctx->mxc_mop->op_id), ctx->mxc_state, 
+                                      __func__, llu(ctx->mxc_mop->op_id), ctx->mxc_state,
                                       peer->mxp_state, mx_strstatus(ctx->mxc_mxstat.code));
                                 bmx_deq_pending_ctx(ctx);
-                                bmx_q_canceled_ctx(ctx,  BMI_ECANCEL);
+                                bmx_q_completed(ctx, BMX_CTX_CANCELED,
+                                                ctx->mxc_mxstat, BMI_ECANCEL);
                         } else {
                                 /* and if not, then disconnect() */
@@ -3182 +3118 @@
                         if (result == 1) {
                                 bmx_deq_pending_ctx(ctx);
-                                bmx_q_canceled_ctx(ctx,  BMI_ECANCEL);
+                                bmx_q_completed(ctx, BMX_CTX_CANCELED,
+                                                BMX_NO_STATUS, BMI_ECANCEL);
                         }
                 }
@@ -3190 +3127 @@
                         ctx->mxc_type == BMX_REQ_TX ? "TX" : "RX", ctx->mxc_state);
         }
+        bmx_release_completion_token();
+
         BMX_EXIT;
 
@@ -3213 +3152 @@
 {
     .method_name               = "bmi_mx",
+    .flags = 0,
     .initialize                = BMI_mx_initialize,
     .finalize                  = BMI_mx_finalize,

branches/Orange-Branch/src/io/bmi/bmi_mx/mx.h

@@ -16 +16 @@
 #include <unistd.h>
 #include <errno.h>
+#include <assert.h>
 
 #include <mx_extensions.h>      /* needed for callback handler, etc. */
@@ -54 +55 @@
 #define BMX_DEBUG     1         /* enable debug (gossip) statements */
 #define BMX_MEM_ACCT  0         /* track number of bytes alloc's and freed */
-#define BMX_LOGGING   0         /* use MPE logging routines */
-
-#if BMX_LOGGING
-#include "mpe.h"
-#endif
 
 #if BMX_MEM_TWEAK
@@ -166 +162 @@
         gen_mutex_t         bmx_idle_rxs_lock;  /* idle_rxs lock */
 
-        list_t              bmx_canceled;       /* canceled reqs waiting for test */
-        gen_mutex_t         bmx_canceled_lock;  /* canceled list lock */
-
-        list_t              bmx_unex_txs;       /* completed unexpected sends */
-        gen_mutex_t         bmx_unex_txs_lock;  /* completed unexpected sends lock */
+        gen_mutex_t         bmx_completion_lock; /* lock for test* functions */
+        int                 bmx_refcount;       /* try to avoid races between test*
+                                                   and cancel functions */
+
+                                                /* completed expected msgs
+                                                 * including unexpected sends */
+        list_t              bmx_done_q[BMI_MAX_CONTEXTS];
+        gen_mutex_t         bmx_done_q_lock[BMI_MAX_CONTEXTS];
+
         list_t              bmx_unex_rxs;       /* completed unexpected recvs */
         gen_mutex_t         bmx_unex_rxs_lock;  /* completed unexpected recvs lock */
@@ -278 +278 @@
         mx_request_t        mxc_mxreq;      /* MX request */
         mx_status_t         mxc_mxstat;     /* MX status */
+        bmi_error_code_t    mxc_error;      /* BMI error code */
 
         uint64_t            mxc_get;        /* # of times returned from idle list */

branches/Orange-Branch/src/io/bmi/bmi_portals/portals.c

@@ -2294 +2294 @@
 {
     .method_name = "bmi_portals",
+    .flags = 0,
     .initialize = bmip_initialize,
     .finalize = bmip_finalize,

branches/Orange-Branch/src/io/bmi/bmi_tcp/socket-collection.c

@@ -90 +90 @@
     if(new_server_socket > -1)
     {
-        tmp_scp->pollfd_array[0].fd = new_server_socket;
-        tmp_scp->pollfd_array[0].events = POLLIN;
-        tmp_scp->addr_array[0] = NULL;
+        tmp_scp->pollfd_array[tmp_scp->array_count].fd = new_server_socket;
+        tmp_scp->pollfd_array[tmp_scp->array_count].events = POLLIN;
+        tmp_scp->addr_array[tmp_scp->array_count] = NULL;
         tmp_scp->array_count++;
-        /* Add the pipe_fd[0] fd to the poll in set always */
-        tmp_scp->pollfd_array[1].fd = tmp_scp->pipe_fd[0];
-        tmp_scp->pollfd_array[1].events = POLLIN;
-        tmp_scp->addr_array[1] = NULL;
-        tmp_scp->array_count++;
-    }
+    }
+
+    /* Add the pipe_fd[0] fd to the poll in set always */
+    tmp_scp->pollfd_array[tmp_scp->array_count].fd = tmp_scp->pipe_fd[0];
+    tmp_scp->pollfd_array[tmp_scp->array_count].events = POLLIN;
+    tmp_scp->addr_array[tmp_scp->array_count] = NULL;
+    tmp_scp->array_count++;
 
     return (tmp_scp);

branches/Orange-Branch/src/io/flow/flow.h

@@ -100 +100 @@
 
     /* function to be triggered upon completion */
-    void(*callback)(struct flow_descriptor* flow_d);
+    void(*callback)(struct flow_descriptor* flow_d, int cancel_path);
 
     struct flow_endpoint src;   /* src endpoint */

branches/Orange-Branch/src/io/flow/flowproto-bmi-cache/flowproto-bmi-cache-server.c

@@ -490 +490 @@
                 free(flow_data);
                 flow_d->release(flow_d);
-                flow_d->callback(flow_d);
+                flow_d->callback(flow_d, 0);
                 fprintf(stderr, "bmi_cache_request_init: exit with return 1. zero request.\n");
                 return(1);
@@ -801 +801 @@
                         flow_d->state = FLOW_COMPLETE;
                         flow_d->release(flow_d);
-                        flow_d->callback(flow_d);
+                        flow_d->callback(flow_d, 0);
                         fprintf(stderr, "bmi_recv_callback_fn: request is done\n");
                         return;

branches/Orange-Branch/src/io/flow/flowproto-bmi-trove/flowproto-multiqueue.c

@@ -31 +31 @@
 #define MAX_REGIONS 64
 
-#define FLOW_CLEANUP(__flow_data)                                     \
+#define FLOW_CLEANUP_CANCEL_PATH(__flow_data, __cancel_path)          \
 do {                                                                  \
     struct flow_descriptor *__flow_d = (__flow_data)->parent;         \
@@ -40 +40 @@
     free(__flow_data);                                                \
     __flow_d->release(__flow_d);                                      \
-    __flow_d->callback(__flow_d);                                     \
+    __flow_d->callback(__flow_d, __cancel_path);                      \
 } while(0)
+
+#define FLOW_CLEANUP(___flow_data) FLOW_CLEANUP_CANCEL_PATH(___flow_data, 0)
 
 struct result_chain_entry
@@ -480 +482 @@
         {
             gen_mutex_unlock(&flow_data->parent->flow_mutex);
-            FLOW_CLEANUP(flow_data);
+            FLOW_CLEANUP_CANCEL_PATH(flow_data, 1);
             return(0);
         }

branches/Orange-Branch/src/io/job/job.c

@@ -130 +130 @@
 static void trove_thread_mgr_callback(void* data,
     PVFS_error error_code);
-static void flow_callback(flow_descriptor* flow_d);
+static void flow_callback(flow_descriptor* flow_d, int cancel_path);
 #ifndef __PVFS2_JOB_THREADED__
 static gen_mutex_t work_cycle_mutex = GEN_MUTEX_INITIALIZER;
@@ -5267 +5267 @@
  * no return value
  */
-static void flow_callback(flow_descriptor* flow_d)
+static void flow_callback(flow_descriptor* flow_d, int cancel_path)
 {
     struct job_desc* tmp_desc = (struct job_desc*)flow_d->user_ptr;
@@ -5281 +5281 @@
 
     /* set job descriptor fields and put into completion queue */
-    gen_mutex_lock(&completion_mutex);
+
+    /* if this is being triggered directly from PINT_flow_cancel(), then the
+     * completion mutex is already held by the caller; skip the mutex.
+     */
+    if(!cancel_path)
+        gen_mutex_lock(&completion_mutex);
     job_desc_q_add(completion_queue_array[tmp_desc->context_id],
                    tmp_desc);
@@ -5295 +5300 @@
     pthread_cond_signal(&completion_cond);
 #endif
-    gen_mutex_unlock(&completion_mutex);
+    if(!cancel_path)
+        gen_mutex_unlock(&completion_mutex);
 
     return;

branches/Orange-Branch/src/io/trove/pvfs2-storage.h

@@ -156 +156 @@
         else                                                   \
         {                                                      \
-            (dest)->mtime = time(NULL);                        \
+            (dest)->mtime = PINT_util_mktime_version(time(NULL)); \
         }                                                      \
     }                                                          \

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-alt-aio.c

@@ -372 +372 @@
     alt_aio_bstream_read_list,
     alt_aio_bstream_write_list,
-    dbpf_bstream_flush
+    dbpf_bstream_flush,
+    NULL
 };
 

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-attr-cache.c

@@ -141 +141 @@
 int dbpf_attr_cache_finalize(void)
 {
-    int ret = -1, i = 0;
+    int ret = -1, i = 0, j = 0;
     struct qlist_head *hash_link = NULL;
     dbpf_attr_cache_elem_t *cache_elem = NULL;
@@ -157 +157 @@
                     cache_elem = qhash_entry(
                         hash_link, dbpf_attr_cache_elem_t, hash_link);
+                    for( j = 0; j < cache_elem->num_keyval_pairs; j++ )
+                    {
+                        /* any elements still existing need their keyval pairs
+                         * data pointer free'd before the element is free'd */
+                        if( cache_elem->keyval_pairs[j].data )
+                            free(cache_elem->keyval_pairs[j].data);
+                    }
                     free(cache_elem);
                     s_current_num_cache_elems--;

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-bstream-direct.c

@@ -659 +659 @@
     PINT_mem_aligned_free(aligned_buf);
 
-    return read_size;
+    return ret;
 }
 
@@ -858 +858 @@
                 goto cache_put;
             }
-            if(qop_p->op.flags & TROVE_SYNC)
-            {
-                sync_required = 1;
-            }
+            sync_required = 1;
         }
         gen_mutex_unlock(&dbpf_update_size_lock);
@@ -1068 +1065 @@
                         NULL,
                         user_ptr,
-                        0,
+                        TROVE_SYNC,
                         context_id);
 
@@ -1354 +1351 @@
             /* consume stream region and update mem region */
             ext_size = cur_stream_size;
+            ext_ptr = cur_mem_off;
+            ext_off = cur_stream_off;
 
             cur_mem_size -= cur_stream_size;

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-dspace.c

@@ -1547 +1547 @@
         }
 
-        return bstream_method_table[method_id]->bstream_cancel(
-            coll_id, id, context_id);
+        if(bstream_method_table[method_id]->bstream_cancel)
+        {
+            return bstream_method_table[method_id]->bstream_cancel(
+                coll_id, id, context_id);
+        }
+        else
+        {
+            gossip_debug(GOSSIP_TROVE_DEBUG, "Trove cancellation is not supported for this operation type; ignoring.\n");
+            return(0);
+
+        }
     }
 

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-mgmt.c

@@ -1954 +1954 @@
     coll_p->meta_sync_enabled = 1; /* MUST be 1 !*/
 
-
     dbpf_collection_register(coll_p);
     *out_coll_id_p = coll_p->coll_id;
+
+    clear_stranded_bstreams(coll_p->coll_id);
+
     return 1;
 }

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-null-aio.c

@@ -395 +395 @@
     null_aio_bstream_read_list,
     null_aio_bstream_write_list,
-    dbpf_bstream_flush
+    dbpf_bstream_flush,
+    NULL
 };
 

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-open-cache.c

@@ -23 +23 @@
 #include <string.h>
 #include <db.h>
+#include <dirent.h>
 
 #include "trove.h"
@@ -634 +635 @@
 }
 
+void clear_stranded_bstreams(TROVE_coll_id coll_id)
+{
+    char path_name[PATH_MAX];
+    DIR *current_dir = NULL;
+    struct dirent *current_dirent = NULL;
+    struct stat file_info;
+    struct file_struct *tmp_item;
+
+    DBPF_GET_STRANDED_BSTREAM_DIRNAME(path_name, PATH_MAX,
+                                      my_storage_p->data_path, coll_id);
+
+    /* remove entries in the stranded bstreams directory */
+    current_dir = opendir(path_name);
+    if(current_dir)
+    {
+        while((current_dirent = readdir(current_dir)))
+        {
+            if((strcmp(current_dirent->d_name, ".") == 0) ||
+               (strcmp(current_dirent->d_name, "..") == 0))
+            {
+                continue;
+            }
+            tmp_item = (struct file_struct *) malloc(sizeof(struct file_struct));
+            if(!tmp_item)
+            {
+                gossip_err("Unable to allocate memory for file_struct [%d].\n", errno);
+                return;
+            }
+            tmp_item->pathname = malloc(PATH_MAX);
+            if(!tmp_item->pathname)
+            {
+                gossip_err("Unable to allocate memory for pathname[%d].\n", errno);
+                free(tmp_item);
+                return;
+            }
+            snprintf(tmp_item->pathname, PATH_MAX, "%s/%s", path_name,
+                     current_dirent->d_name);
+            if(stat(tmp_item->pathname, &file_info) < 0)
+            {
+                gossip_err("error doing stat on bstream entry\n");
+                continue;
+            }
+            assert(S_ISREG(file_info.st_mode));
+            /* Add to the queue */
+
+            pthread_mutex_lock(&dbpf_unlink_context.mutex);
+            qlist_add_tail(&tmp_item->list_link, &dbpf_unlink_context.global_list);
+            pthread_cond_signal(&dbpf_unlink_context.data_available);
+            pthread_mutex_unlock(&dbpf_unlink_context.mutex);
+            gossip_debug(GOSSIP_DBPF_OPEN_CACHE_DEBUG,
+              "Added [%s] to the queue.\n", tmp_item->pathname);
+        }
+        closedir(current_dir);
+    }
+    else
+    {   
+        gossip_err("Unable to open stranded bstream directory [%s] to "
+          "perform initialization of stranded bstream cleanup", path_name);
+    }
+}
+
 /*
  * Local variables:

branches/Orange-Branch/src/io/trove/trove-dbpf/dbpf-open-cache.h

@@ -45 +45 @@
     TROVE_handle handle);
 
+void clear_stranded_bstreams(TROVE_coll_id coll_id);
+    
 #endif /* __DBPF_OPEN_CACHE_H__ */
 

branches/Orange-Branch/src/kernel/linux-2.6/inode.c

@@ -134 +134 @@
 struct backing_dev_info pvfs2_backing_dev_info =
 {
+#ifdef HAVE_BACKING_DEV_INFO_NAME
+    .name = "pvfs2",
+#endif
     .ra_pages = 0,
 #ifdef HAVE_BDI_MEMORY_BACKED

branches/Orange-Branch/src/kernel/linux-2.6/namei.c

@@ -71 +71 @@
     pvfs2_inode_t *parent = NULL, *found_pvfs2_inode = NULL;
     struct super_block *sb = NULL;
-    struct dentry *res = NULL;
 
     /*
@@ -202 +201 @@
     if (inode && !is_bad_inode(inode))
     {
+        struct dentry *res;
+
         gossip_debug(GOSSIP_NAME_DEBUG, "%s:%s:%d Found good inode [%lu] with count [%d]\n", 
             __FILE__, __func__, __LINE__, ino