Changeset 9025

Timestamp:

08/23/11 15:40:35 (21 months ago)

Author:

jdenton

Message:

Integrating ucache with usrint (in progress):

Added a parameter to the pvfs_alloc_descriptor function in openfile-util:

the parameter is a pointer to a PVFS_object_ref.
the parameter can be NULL or it is used by the ucache to identify the file being inserted.

Added calls to ucache in openfile-util.c:

specifically in functions:

pvfs_alloc_descriptor
pvfs_free descriptor

ucache updates include the introduction of block level locks which may be needed

Location:

branches/Orange-Branch/src/client/usrint

Files:

• iocommon.c (modified) (6 diffs)
• iocommon.h (modified) (3 diffs)
• openfile-util.c (modified) (5 diffs)
• openfile-util.h (modified) (1 diff)
• posix-ops.h (modified) (1 diff)
• posix.c (modified) (1 diff)
• socket.c (modified) (4 diffs)
• ucache.c (modified) (10 diffs)
• ucache.h (modified) (3 diffs)

branches/Orange-Branch/src/client/usrint/iocommon.c

@@ -418 +418 @@
             struct stat sbuf;
             /* try to open using glibc */
-            rc = (*glibc_ops.open)(error_path, flags & 01777777, mode);
+            rc = (*glibc_ops.open)(error_path, fpvfs_alloc_descriptorlags & 01777777, mode);
             IOCOMMON_RETURN_ERR(rc);
-            pd = pvfs_alloc_descriptor(&pvfs_ops, -1);
+            pd = pvfs_alloc_descriptor(&pvfs_ops, -1, NULL);
             pd->is_in_use = PVFS_FS;    /* indicate fd is valid! */
             pd->true_fd = rc;
@@ -462 +462 @@
         }
     }
+
     /* If we get here the file was created and/or opened */
-
     /* Translate the pvfs reference into a file descriptor */
     /* Set the file information */
     /* create fd object */
-    pd = pvfs_alloc_descriptor(&pvfs_ops, -1);
+    pd = pvfs_alloc_descriptor(&pvfs_ops, -1, &file_ref);
     if (!pd)
     {
@@ -836 +836 @@
 }
 
+/*The Wrapper Fuction calls to the "nocache" version of 
+ *  io_common_readorwrite (below)   
+ */
+/** do a blocking read or write, possibly utilizing the user cache.
+ * 
+ */
+int iocommon_readorwrite(enum PVFS_io_type which,
+                         pvfs_descriptor *pd,
+                         PVFS_size offset,
+                         void *buf,
+                         PVFS_Request mem_req,
+                         PVFS_Request file_req)
+{
+    #ifndef UCACHE_ENABLED
+    /* No cache */
+    return iocommon_readorwrite_nocache(which, pd, offset, buf, mem_req,
+                                                                file_req);
+    #endif /* UCACHE_ENABLED */
+
+    //read
+    //readthedata(*filePtrToData)
+    //write
+
+
+    //Cache Routines
+    //Possibly Data Transfer
+    //Possibly More Cache Routines
+}
+
 /** do a blocking read or write
  * 
  */
-int iocommon_readorwrite(enum PVFS_io_type which,
+int iocommon_readorwrite_nocache(enum PVFS_io_type which,
                          pvfs_descriptor *pd,
                          PVFS_size offset,
@@ -886 +915 @@
 }
 
+/** Do a nonblocking read or write, possibly utilizing the user cache.
+ *
+ * extra_offset = extra padding to the pd's offset,
+ * independent of the pd's offset
+ * Returns an op_id, response, and ret_mem_request
+ * (which represents an etype_req*count region)
+ * Note that the none of the PVFS_Requests are freed
+ */
+int iocommon_ireadorwrite(enum PVFS_io_type which,
+                          pvfs_descriptor *pd,
+                          PVFS_size extra_offset,
+                          void *buf,
+                          PVFS_Request etype_req,
+                          PVFS_Request file_req,
+                          size_t count,
+                          PVFS_sys_op_id *ret_op_id,
+                          PVFS_sysresp_io *ret_resp,
+                          PVFS_Request *ret_memory_req)
+{
+    #ifndef UCACHE_ENABLED
+    /* No cache */
+    return iocommon_ireadorwrite_nocache(which, pd, extra_offset, buf, 
+        etype_req, file_req, count, ret_op_id, ret_resp, ret_memory_req);
+    #endif /* UCACHE_ENABLED */
+
+    //if read then check cache..if not there..then read from i/o node and store into correct location
+    //Possibly Data Transfer
+    //Possibly More Cache Routines
+}
+
+
 /** Do a nonblocking read or write
  *
@@ -894 +954 @@
  * Note that the none of the PVFS_Requests are freed
  */
-int iocommon_ireadorwrite(enum PVFS_io_type which,
+int iocommon_ireadorwrite_nocache(enum PVFS_io_type which,
                           pvfs_descriptor *pd,
                           PVFS_size extra_offset,
@@ -1839 +1899 @@
 }
 
-/** Implelments an extended attribute delete or remove
+/** Implements an extended attribute delete or remove
  *
  *  The PVFS server enforces namespaces as prefixes on the

branches/Orange-Branch/src/client/usrint/iocommon.h

@@ -104 +104 @@
                     PVFS_object_ref *newpdir, const char *newname);
 
+/* R/W Wrapper Functions, possibly utilizing user cache */
 /* do a blocking read or write
- * extra_offset = extra padding to the pd's offset, independent of the pd's offset */
+ */
 int iocommon_readorwrite(enum PVFS_io_type which,
                          pvfs_descriptor *pd,
@@ -112 +113 @@
                          PVFS_Request mem_req,
                          PVFS_Request file_req);
-        //returned by nonblocking operations
-
-/*
- * [Do a nonblocking read or write]
- * extra_offset = extra padding to the pd's offset,
- * independent of the pd's offset
- * Returns an op_id, response, and ret_mem_request
- * (which represents an etype_req*count region)
- * Note that the none of the PVFS_Requests are freed
- */
-int iocommon_ireadorwrite(enum PVFS_io_type which,
+
+
+/* [Do a nonblocking read or write] */
+int iocommon_ireadorwrite_nocache(enum PVFS_io_type which,
                           pvfs_descriptor *pd,
                           PVFS_size extra_offset,
@@ -133 +127 @@
                           PVFS_Request *ret_memory_req);
 
+
+/* do a blocking read or write
+ * extra_offset = extra padding to the pd's offset, independent of the pd's offset */
+int iocommon_readorwrite_nocache(enum PVFS_io_type which,
+                         pvfs_descriptor *pd,
+                         PVFS_size offset,
+                         void *buf,
+                         PVFS_Request mem_req,
+                         PVFS_Request file_req);
+        //returned by nonblocking operations
+
+/*
+ * [Do a nonblocking read or write]
+ * extra_offset = extra padding to the pd's offset,
+ * independent of the pd's offset
+ * Returns an op_id, response, and ret_mem_request
+ * (which represents an etype_req*count region)
+ * Note that the none of the PVFS_Requests are freed
+ */
+int iocommon_ireadorwrite_nocache(enum PVFS_io_type which,
+                          pvfs_descriptor *pd,
+                          PVFS_size extra_offset,
+                          void *buf,
+                          PVFS_Request etype_req,
+                          PVFS_Request file_req,
+                          size_t count,
+                          PVFS_sys_op_id *ret_op_id,
+                          PVFS_sysresp_io *ret_resp,
+                          PVFS_Request *ret_memory_req);
+
 int iocommon_getattr(PVFS_object_ref obj, PVFS_sys_attr *attr, uint32_t mask);
 

branches/Orange-Branch/src/client/usrint/openfile-util.c

@@ -16 +16 @@
 #include <openfile-util.h>
 #include <posix-pvfs.h>
+#include <ucache.h>
 
 static struct glibc_redirect_s
@@ -396 +397 @@
  * initialize fsops to the given set
  */
- pvfs_descriptor *pvfs_alloc_descriptor(posix_ops *fsops, int fd)
+ pvfs_descriptor *pvfs_alloc_descriptor(posix_ops *fsops, int fd, 
+                                        PVFS_object_ref *file_ref
+ )
  {
         int newfd, rc, flags; 
@@ -443 +446 @@
         descriptor_table[newfd]->dpath = NULL;
         descriptor_table[newfd]->is_in_use = PVFS_FS;
+    descriptor_table[newfd]->mtbl = NULL;
+
+    /* File reference won't always be passed in */
+    if(file_ref != NULL)
+    {
+        /* We have the file identifiers, so insert file info into ucache */
+        rc = ucache_open_file(&(file_ref->fs_id), &(file_ref->handle), 
+                                       descriptor_table[newfd]->mtbl);
+        /* Unique Entry */
+        if(rc > 0)
+        {   
+            descriptor_table[newfd]->mtbl->ref_cnt = 1;
+        }
+        /* File already in Cache */
+        else if(rc == 0){
+            descriptor_table[newfd]->mtbl->ref_cnt++;
+        }
+        /* Could not insert */
+        else
+        {
+            /* TODO: probably need modify this */  
+        }
+    }
 
     return descriptor_table[newfd];
@@ -519 +545 @@
             pd->fsops = &glibc_ops;
             pd->true_fd = fd;
-            pd->pvfs_ref.fs_id = 0;
+            pd->.fs_id = 0;
             pd->pvfs_ref.handle = 0;
             pd->flags = flags;
@@ -562 +588 @@
             free(pd->dpath);
         }
-        /* release cache opbjects here */
+        /* release cache objects here */
+        pd->mtbl->ref_cnt--;
+        if(pd->mtbl->ref_cnt == 0)
+        {
+            /* Flush dirty blocks before file removal from cache */
+            ucache_flush(&(pd->pvfs_ref.fs_id), &(pd->pvfs_ref.handle));
+            /* remove all of this file's associated data from cache */
+            ucache_close_file(&(pd->pvfs_ref.fs_id), &(pd->pvfs_ref.handle));
+        }
+
             /* free descriptor - wipe memory first */
             memset(pd, 0, sizeof(pvfs_descriptor));

branches/Orange-Branch/src/client/usrint/openfile-util.h

@@ -44 +44 @@
                             PVFS_object_ref *ref);
 
-pvfs_descriptor *pvfs_alloc_descriptor(posix_ops *fsops, int fd);
+pvfs_descriptor *pvfs_alloc_descriptor(posix_ops *fsops, 
+                                                int fd, 
+                                                PVFS_object_ref *file_ref);
 
 pvfs_descriptor *pvfs_find_descriptor(int fd);

branches/Orange-Branch/src/client/usrint/posix-ops.h

@@ -165 +165 @@
     int is_in_use;            /**< PVFS_FS if this descriptor is valid */
     char *dpath;              /**< path of an open directory for fchdir */
+    struct mem_table_s *mtbl; /**< reference to cached objects */            
 } pvfs_descriptor;
 

branches/Orange-Branch/src/client/usrint/posix.c

@@ -69 +69 @@
         }
         /* set up the descriptor manually */
-        pd = pvfs_alloc_descriptor(&glibc_ops, rc);
+        pd = pvfs_alloc_descriptor(&glibc_ops, rc, NULL);
         if (!pd)
         {

branches/Orange-Branch/src/client/usrint/socket.c

@@ -30 +30 @@
         return sockfd;
     }
-    pd = pvfs_alloc_descriptor(&glibc_ops, sockfd);
+    pd = pvfs_alloc_descriptor(&glibc_ops, sockfd, NULL);
     pd->mode |= S_IFSOCK;
     return pd->fd;
@@ -59 +59 @@
         goto errorout;
     }
-    pd = pvfs_alloc_descriptor(&glibc_ops, fd);
+    pd = pvfs_alloc_descriptor(&glibc_ops, fd , NULL);
     pd->mode |= S_IFSOCK;
     rc = fd;   
@@ -436 +436 @@
         goto errorout;
     }
-    pd0 = pvfs_alloc_descriptor(&glibc_ops, sv[0]);
+    pd0 = pvfs_alloc_descriptor(&glibc_ops, sv[0], NULL);
     if (!pd0)
     {
         goto errorout;
     }
-    pd1 = pvfs_alloc_descriptor(&glibc_ops, sv[1]);
+    pd1 = pvfs_alloc_descriptor(&glibc_ops, sv[1], NULL);
     if (!pd1)
     {
@@ -473 +473 @@
         goto errorout;
     }
-    f0 = pvfs_alloc_descriptor(&glibc_ops, fa[0]);
+    f0 = pvfs_alloc_descriptor(&glibc_ops, fa[0], NULL);
     if (!f0)
     {
         goto errorout;
     }
-    f1 = pvfs_alloc_descriptor(&glibc_ops, fa[1]);
+    f1 = pvfs_alloc_descriptor(&glibc_ops, fa[1], NULL);
     if (!f1)
     {

branches/Orange-Branch/src/client/usrint/ucache.c

@@ -18 +18 @@
 
 /* Global Variables */
+static FILE *out;                   /* For Logging Purposes */
+
 static union user_cache_u *ucache;
 static int ucache_blk_cnt;
-static ucache_lock_t *ucache_lock;  /* For maintaining concurrency */
-static FILE *out;                   /* For Logging Purposes */
+
+static ucache_lock_t *ucache_locks; /* will refer to the shmem of all ucache locks */
+static ucache_lock_t *ucache_lock;  /* Global Lock maintaining concurrency */
+static ucache_lock_t *ucache_block_lock;
 
 /* Internal Only Function Declarations */
 
-/* Global Cache Lock */
+/* Locking */
 static int lock_init(ucache_lock_t * lock);
 static int lock_lock(ucache_lock_t * lock);
@@ -112 +116 @@
 void ucache_initialize(void)
 {
-    /* Aquire shared memory for ucache_lock */
-    ucache_lock = shmat(shmget(ftok(GET_KEY_FILE, 'a'), 
-        sizeof(ucache_lock_t), CACHE_FLAGS), NULL, AT_FLAGS);
+    int i = 0;
+
+    /* Aquire shared memory for ucache_locks */
+    ucache_locks = shmat(shmget(ftok(GET_KEY_FILE, 'a'), (LOCK_SIZE 
+           * (BLOCKS_IN_CACHE + 1)), CACHE_FLAGS), NULL, AT_FLAGS);
+    /* Global Cache lock stored in first LOCK_SIZE position */
+    ucache_lock = ucache_locks; 
+
+    /* Initialize Global Cache Lock */
     lock_init(ucache_lock);
     lock_lock(ucache_lock);
-    #if LOCK_TYPE==0
-    int lockVal;
-    ucache_lock_getvalue(ucache_lock, &lockVal);
-    printf("lock value = %d\n", lockVal);
-    #endif
-    /* Aquire shared memory for lock for ucache */
-    int key, id, i;
+
+    /* The next BLOCKS_IN_CACHE number of block locks follow the global lock */
+    ucache_block_lock = ucache_locks + 1;
+    /* Initialize Block Level Locks */
+    for(i = 0; i < BLOCKS_IN_CACHE; i++)
+    {
+        lock_init(get_block_lock(i));
+    }
+
+    /* Aquire shared memory for ucache */
+    int key;
+    int id;
     char *key_file_path;
     /*  Direct output   */
@@ -172 +187 @@
 }
 
-int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle)
+int ucache_open_file(PVFS_fs_id *fs_id, PVFS_handle *handle, 
+                                    struct mem_table_s *mtbl
+)
+{
+    lock_lock(ucache_lock);
+    mtbl= lookup_file((uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL,
+                                                                          NULL);
+    if((int)mtbl==NIL)
+    {
+        mtbl = insert_file((uint32_t)*fs_id, (uint64_t)*handle);
+        if((int)mtbl==NIL)
+        {   /* Error - Could not insert */
+            lock_unlock(ucache_lock);
+            return -1;
+        }
+        else
+        {
+            /* File Inserted*/
+            lock_unlock(ucache_lock);
+            return 1;
+        }
+    }
+    else
+    {
+        /* File was previously Inserted */
+        lock_unlock(ucache_lock);
+        return 0;
+    }
+}
+
+/** Returns ptr to block in cache based on file and offset */
+void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset)
+{
+    lock_lock(ucache_lock);
+    struct mem_table_s *mtbl= lookup_file(
+        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL);
+    if((int)mtbl!=NIL)
+    {
+        char *retVal = (char *)lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, 
+                                                                        NULL);
+        lock_unlock(ucache_lock);
+        return((void *)retVal); 
+    }
+    lock_unlock(ucache_lock);
+    return (void *)NIL;
+}
+
+/** Prepares the data structures for block storage. 
+ * On success, returns a pointer to where the block of data should be written. 
+ * On failure, returns NIL.
+ */
+void *ucache_insert(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset)
 {
     lock_lock(ucache_lock);
@@ -179 +245 @@
     if((int)mtbl==NIL)
     {
-        insert_file((uint32_t)*fs_id, (uint64_t)*handle);
-    }
-    lock_unlock(ucache_lock);
-    return 1;
-}
-
-/** Returns ptr to block in cache based on file and offset */
-void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset)
-{
-    lock_lock(ucache_lock);
-    struct mem_table_s *mtbl= lookup_file(
-        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL);
-    if((int)mtbl!=NIL)
-    {
-        char *retVal = (char *)lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, 
-                                                                        NULL);
-        lock_unlock(ucache_lock);
-        return((void *)retVal); 
-    }
-    lock_unlock(ucache_lock);
-    return (void *)NIL;
-}
-
-/** Prepares the data structures for block storage. 
- * On success, returns a pointer to where the block of data should be written. 
- * On failure, returns NIL.
- */
-void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset)
-{
-    lock_lock(ucache_lock);
-    struct mem_table_s *mtbl= lookup_file(
-        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL);
-    if((int)mtbl==NIL)
-    {
         lock_unlock(ucache_lock);
         return (void *)NIL;
@@ -229 +261 @@
 
 /**  Removes a cached block of data from mtbl */
-int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset)
+int ucache_remove(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset)
 {
     lock_lock(ucache_lock);
@@ -245 +277 @@
 
 /** Flushes dirty blocks to the I/O Nodes */
-int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle)
+int ucache_flush(PVFS_fs_id *fs_id, PVFS_handle *handle)
 {
     lock_lock(ucache_lock);
@@ -273 +305 @@
  * the cache.
  */
-int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle)
+int ucache_close_file(PVFS_fs_id *fs_id, PVFS_handle *handle)
 {
     lock_lock(ucache_lock);
@@ -474 +506 @@
     }
 }
+
+/** Returns a pointer to the block level lock corresponding to the block_index.
+ */
+ucache_lock_t * get_block_lock(int block_index)
+{
+    return (ucache_block_lock + block_index);
+}
+
 /***************************************** End of Externally Visible API */
 
@@ -522 +562 @@
  * Otherwise, returns -1 and sets errno.
  */
-#if LOCK_TYPE==0
+#if (LOCK_TYPE == 0)
 int ucache_lock_getvalue(ucache_lock_t * lock, int *sval)
 {
@@ -534 +574 @@
 static int lock_destroy(ucache_lock_t * lock)
 {
-    #if LOCK_TYPE==0
+    #if (LOCK_TYPE == 0)
     return sem_destroy(lock); 
-    #elif LOCK_TYPE==1
+    #elif (LOCK_TYPE == 1)
     return pthread_mutex_destroy(lock);
-    #elif LOCK_TYPE==2
+    #elif (LOCK_TYPE == 2)
     return pthread_spin_destroy(lock);
     #endif

branches/Orange-Branch/src/client/usrint/ucache.h

@@ -40 +40 @@
 #define NIL (-1)
 
-#define DBG 1
+#define DBG 0
 #define INTERNAL_TESTING 0
 
 
-#define LOCK_TYPE 0 /* 0 for Semaphore, 1 for Mutex, 2 for Spinlock */
-#if LOCK_TYPE==0
+#define LOCK_TYPE 1 /* 0 for Semaphore, 1 for Mutex, 2 for Spinlock */
+#if (LOCK_TYPE == 0)
 #define ucache_lock_t sem_t
-#elif LOCK_TYPE==1
-#define ucache_lock_t pthread_mutex_t
-#elif LOCK_TYPE==2
+#define LOCK_SIZE sizeof(sem_t)
+#elif (LOCK_TYPE == 1)
+#define ucache_lock_t pthread_mutex_t /* sizeof(pthread_mutex_t)=24 */
+#define LOCK_SIZE sizeof(pthread_mutex_t)
+#elif (LOCK_TYPE == 2)
 #define ucache_lock_t pthread_spinlock_t
+#define LOCK_SIZE sizeof(pthread_spinlock_t)
 #endif
 
 typedef uint32_t PVFS_fs_id;
-typedef uint64_t PVFS_object_ref;
+typedef uint64_t PVFS_handle;
 
 /** A link for one block of memory in a files hash table
@@ -134 +137 @@
 };
 
+struct ucache_ref_s
+{
+    union user_cache_u *ucache;           /* pointer to ucache shmem */
+    ucache_lock_t *ucache_locks;    /* pointer to ucache locks */
+};
+
 /* externally visible API */
 void ucache_initialize(void);
-int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle);
-void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset);
-void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset);
-int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset);
-int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle);
-int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle);
+int ucache_open_file(PVFS_fs_id *fs_id, PVFS_handle *handle, 
+                                  struct mem_table_s *mtbl);
+int ucache_close_file(PVFS_fs_id *fs_id, PVFS_handle *handle);
+void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset);
+void *ucache_insert(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset);
+int ucache_remove(PVFS_fs_id *fs_id, PVFS_handle *handle, uint64_t offset);
+int ucache_flush(PVFS_fs_id *fs_id, PVFS_handle *handle);
 void ucache_dec_ref_cnt(struct mem_table_s * mtbl);
 void ucache_inc_ref_cnt(struct mem_table_s * mtbl);
@@ -149 +159 @@
     ucache_lock_t * ucache_lock
 );
+ucache_lock_t * get_block_lock(int block_index);
 
 #if LOCK_TYPE==0
 int ucache_lock_getvalue(ucache_lock_t * lock, int *sval);
-#endif
+#endif /* LOCK_TYPE */
 /****************************************  End of Internal Only Functions    */
-#endif
+#endif /* UCACHE_H */
 /*
  * Local variables: