Changeset 9001

Show
Ignore:
Timestamp:
08/09/11 13:04:46 (22 months ago)
Author:
jdenton
Message:

Added a cache locking feature utilized by the externally visible API.
Locking functions are declared static to prohibit the user from calling
them to prevent potential cache corruption due to multiprocessing.

Location:
branches/Orange-Branch/src/client/usrint
Files:
2 modified

Legend:

Unmodified
Added
Removed
  • branches/Orange-Branch/src/client/usrint/ucache.c

    r8977 r9001  
    55 */ 
    66 
     7/*  Experimental cache for user data 
     8 *      Currently under development. 
     9 */ 
     10 
     11/*   
     12 *   Note: When unsigned ints are set to NIL, their values are based on type: 
     13 *   ex: 16      0xFFFF   
     14 *       32      0XFFFFFFFF 
     15 *       64      0XFFFFFFFFFFFFFFFF  
     16 *   ALL EQUAL THE SIGNED REPRESENTATION OF -1, CALLED NIL.   
     17 */ 
     18 
    719#include "ucache.h" 
    8  
    9 /*   
    10 *   Note: When unsigned ints are set to NIL, their values are based on type: 
    11 *   ex: 16      0xFFFF   
    12 *       32      0XFFFFFFFF 
    13 *       64      0XFFFFFFFFFFFFFFFF  
    14 *   ALL EQUAL THE SIGNED REPRESENTATION OF -1, CALLED NIL.   
    15 */ 
    1620 
    1721static union user_cache_u *ucache; 
    1822static int ucache_blk_cnt; 
    19 FILE *out; /*   For Logging Purposes    */ 
    20  
    21 static union user_cache_u *get_ucache(){ 
    22     return ucache; 
    23 } 
    24  
    25 /*  This function should only be called when the ftbl has no free mtbls */ 
    26 static void add_free_mtbls(int blk) 
    27 { 
    28     int i, start_mtbl; 
    29     struct file_table_s *ftbl = &(ucache->ftbl); 
    30     union cache_block_u *b = &(ucache->b[blk]); 
    31  
    32     /* add mtbls in blk to ftbl free list */ 
    33     if (blk == 0) 
    34     { 
    35         start_mtbl = 1; /* skip blk 0 ent 0 which is ftbl */ 
    36     } 
    37     else 
    38     { 
    39         start_mtbl = 0; 
    40     } 
    41     for (i = start_mtbl; i < MTBL_PER_BLOCK - 1; i++) 
    42     { 
    43         b->mtbl[i].free_list_blk = blk; 
    44         b->mtbl[i].free_list = i + 1; 
    45     } 
    46     b->mtbl[i].free_list_blk = NIL; 
    47     b->mtbl[i].free_list = NIL; 
    48     ftbl->free_mtbl_blk = blk; 
    49     ftbl->free_mtbl_ent = start_mtbl; 
    50 } 
    51  
    52 void ucache_initialize(void) 
    53 { 
     23static ucache_lock_t *ucache_lock;  /*  For maintaining concurrency */ 
     24static FILE *out;   /*  For Logging Purposes    */ 
     25 
     26/*  Externally Visible API 
     27 *      The following functions are thread/processor safe regarding the cache  
     28 *      tables and data.       
     29 */ 
     30 
     31/*  Initializes the cache.  
     32 *  Mainly, it aquires a shared memory segment used to cache data.  
     33 *  
     34 *  This function also initializes the the FTBL and some MTBLs. 
     35 * 
     36 *  The whole cache is protected by a locking mechanism to maintain concurrency. 
     37 *  Currently using posix semaphores 
     38 */ 
     39extern void ucache_initialize(void) 
     40{ 
     41    /*  Aquire shared memory for ucache_lock    */ 
     42    ucache_lock = shmat(shmget(ftok(GET_KEY_FILE, 'a'),  
     43        sizeof(ucache_lock_t), CACHE_FLAGS), NULL, AT_FLAGS); 
     44    ucache_lock_init(ucache_lock); 
     45    ucache_lock_lock(ucache_lock); 
     46    /*  Aquire shared memory for lock for ucache    */ 
    5447    int key, id, i; 
    5548    char *key_file_path; 
     
    9689    } 
    9790    ucache->ftbl.file[FILE_TABLE_ENTRY_COUNT - 1].next = NIL; 
     91    ucache_lock_unlock(ucache_lock); 
     92} 
     93 
     94extern int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
     95{ 
     96    ucache_lock_lock(ucache_lock); 
     97    struct mem_table_s *mtbl= lookup_file( 
     98        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
     99    if((int)mtbl==NIL) 
     100    { 
     101        insert_file((uint32_t)*fs_id, (uint64_t)*handle); 
     102    } 
     103    ucache_lock_unlock(ucache_lock); 
     104    return 1; 
     105} 
     106 
     107/*  Returns ptr to block in cache based on file and offset  */ 
     108extern void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
     109{ 
     110    ucache_lock_lock(ucache_lock); 
     111    struct mem_table_s *mtbl= lookup_file( 
     112        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
     113    if((int)mtbl!=NIL) 
     114    { 
     115        char *retVal = (char *)lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, NULL); 
     116        ucache_lock_unlock(ucache_lock); 
     117        return((void *)retVal);  
     118    } 
     119    ucache_lock_unlock(ucache_lock); 
     120    return (void *)NIL; 
     121} 
     122 
     123/*  Inserts block of data into cache    */ 
     124extern void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
     125{ 
     126    ucache_lock_lock(ucache_lock); 
     127    struct mem_table_s *mtbl= lookup_file( 
     128        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
     129    if((int)mtbl==NIL) 
     130    { 
     131        ucache_lock_unlock(ucache_lock); 
     132        return (void *)NIL; 
     133    } 
     134    else 
     135    { 
     136        remove_mem(mtbl, (uint64_t)offset); 
     137        char * retVal= insert_mem(mtbl, (uint64_t)offset); 
     138        ucache_lock_unlock(ucache_lock); 
     139        return ((void *)retVal);  
     140    } 
     141} 
     142 
     143/*  Removes a cached block of data from mtbl    */ 
     144extern int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
     145{ 
     146    ucache_lock_lock(ucache_lock); 
     147    struct mem_table_s *mtbl= lookup_file( 
     148        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
     149    if((int)mtbl!=NIL) 
     150    { 
     151        int retVal = remove_mem(mtbl, (uint64_t)offset); 
     152        ucache_lock_unlock(ucache_lock);  
     153        return retVal;  
     154    }         
     155    ucache_lock_unlock(ucache_lock); 
     156    return NIL; 
     157} 
     158 
     159/*  Flushes dirty blocks to the I/O Nodes   */ 
     160extern int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
     161{ 
     162    ucache_lock_lock(ucache_lock); 
     163    struct mem_table_s *mtbl= lookup_file( 
     164        (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
     165    if((int)mtbl==NIL) 
     166    { 
     167        return NIL; 
     168    } 
     169    int i; 
     170    for(i=mtbl->dirty_list; !dirty_done(i); i=dirty_next(mtbl, i)){ 
     171        struct mem_ent_s *ment = &(mtbl->mem[i]); 
     172        mtbl->mem[i].dirty_next = NIL; 
     173        if((int64_t)ment->tag==NIL || (int32_t)ment->item==NIL){ 
     174            break; 
     175        } 
     176        /*  //flush block to disk   */ 
     177    } 
     178    mtbl->dirty_list = NIL; 
     179    ucache_lock_unlock(ucache_lock); 
     180    return 1; 
     181} 
     182 
     183/*  Removes all memory entries in the mtbl corresponding to the file info  
     184 *  provided as parameters. It also removes the mtbl and the file entry from  
     185 *  the cache. 
     186 */ 
     187extern int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
     188{ 
     189    ucache_lock_lock(ucache_lock); 
     190    uint32_t file_mtbl_blk; 
     191    uint16_t file_mtbl_ent; 
     192    uint16_t file_ent_index; 
     193    uint16_t file_ent_prev_index; 
     194    struct mem_table_s *mtbl= lookup_file( 
     195        (uint32_t)(*fs_id),  
     196        (uint64_t)(*handle),  
     197        &file_mtbl_blk,  
     198        &file_mtbl_ent,  
     199        &file_ent_index,  
     200        &file_ent_prev_index); 
     201    if((int)mtbl==NIL) 
     202    { 
     203        ucache_lock_unlock(ucache_lock); 
     204        return NIL; 
     205    } 
     206    remove_all_memory_entries(mtbl); 
     207    struct file_ent_s *file = &(ucache->ftbl.file[file_ent_index]); 
     208    put_free_mtbl(mtbl, file); 
     209    put_free_fent(file_ent_index); 
     210    ucache_lock_unlock(ucache_lock); 
     211    return 1; 
     212} 
     213 
     214/*  Use the following function to decrement the reference count of a particular  
     215 *  mtbl. This function must be called when a user's code is done with the 
     216 *  pointer provided via any of the pointer returning external functions listed 
     217 *  above. 
     218 */ 
     219extern int ucache_dec_ref_cnt(struct mem_table_s * mtbl) 
     220{ 
     221    ucache_lock_lock(ucache_lock); 
     222    /*  decrement ref_cnt of mtbl   */ 
     223    mtbl->ref_cnt--; 
     224    ucache_lock_unlock(ucache_lock);     
     225} 
     226/*****************************************  End of Externally Visible API    */ 
     227 
     228 
     229/*  Beginning of internal only (static) functions 
     230 */ 
     231 
     232/*  Internally Available Locking Mechanism - using POSIX semaphores    
     233 */ 
     234static int ucache_lock_init(ucache_lock_t * lock) 
     235{ 
     236    /*  Set pshared (2nd arg) to non-zero value to share semaphore b/w forked  
     237     *  processes 
     238     */ 
     239    return sem_init(lock, 1, 1);  
     240} 
     241 
     242/*  Returns 0 when lock is locked; otherwise, return -1 and sets errno*/ 
     243static int ucache_lock_lock(ucache_lock_t * lock) 
     244{ 
     245    return sem_wait(lock); 
     246} 
     247 
     248/*  If successful, return zero; otherwise, return -1 and sets errno */ 
     249static int ucache_lock_unlock(ucache_lock_t * lock) 
     250{ 
     251    return sem_post(lock); 
     252} 
     253 
     254/*  Upon successful completion, returns zero; otherwise, returns and sets errno. 
     255 */ 
     256static int ucache_lock_getvalue(ucache_lock_t * lock, int *sval) 
     257{ 
     258    return sem_getvalue(lock, sval); 
     259} 
     260 
     261/* Upon successful completion, returns zero; otherwise returns 1 and sets errno. 
     262 */ 
     263static int ucache_lock_destroy(ucache_lock_t * lock) 
     264{ 
     265    return sem_destroy(lock);  
     266} 
     267 
     268/*  Dirty List Iterator */ 
     269static int dirty_done(uint16_t index) 
     270{ 
     271    return ((int16_t)index==NIL); 
     272} 
     273 
     274static int dirty_next(struct mem_table_s *mtbl, uint16_t index) 
     275{ 
     276    return mtbl->mem[index].dirty_next; 
     277} 
     278 
     279/*  Memory Entry Chain Iterator */ 
     280static int ment_done(int index) 
     281{ 
     282    return ((int16_t)index==NIL); 
     283} 
     284 
     285static int ment_next(struct mem_table_s *mtbl, int index) 
     286{ 
     287    return mtbl->mem[index].next; 
     288} 
     289 
     290/*  File Entry Chain Iterator   */ 
     291static int file_done(int index) 
     292{ 
     293    return ((int16_t)index==NIL); 
     294} 
     295 
     296static int file_next(struct file_table_s *ftbl, int index) 
     297{ 
     298    return ftbl->file[index].next; 
     299} 
     300 
     301/*   
     302 *  This function should only be called when the ftbl has no free mtbls.  
     303 */ 
     304static void add_free_mtbls(int blk) 
     305{ 
     306    int i, start_mtbl; 
     307    struct file_table_s *ftbl = &(ucache->ftbl); 
     308    union cache_block_u *b = &(ucache->b[blk]); 
     309 
     310    /* add mtbls in blk to ftbl free list */ 
     311    if (blk == 0) 
     312    { 
     313        start_mtbl = 1; /* skip blk 0 ent 0 which is ftbl */ 
     314    } 
     315    else 
     316    { 
     317        start_mtbl = 0; 
     318    } 
     319    for (i = start_mtbl; i < MTBL_PER_BLOCK - 1; i++) 
     320    { 
     321        b->mtbl[i].free_list_blk = blk; 
     322        b->mtbl[i].free_list = i + 1; 
     323    } 
     324    b->mtbl[i].free_list_blk = NIL; 
     325    b->mtbl[i].free_list = NIL; 
     326    ftbl->free_mtbl_blk = blk; 
     327    ftbl->free_mtbl_ent = start_mtbl;    
    98328} 
    99329 
     
    228458    struct file_table_s *ftbl = &(ucache->ftbl); 
    229459    struct file_ent_s *current; /* Current ptr for iteration    */ 
    230     int index;          /* Index into file hash table   */ 
    231     index = handle % FILE_TABLE_HASH_MAX; 
     460    /* Index into file hash table   */ 
     461    int index = handle % FILE_TABLE_HASH_MAX; 
    232462    if(DBG)fprintf(out, "\thashed index: %d\n", index); 
    233463    current = &(ftbl->file[index]); 
     
    295525} 
    296526 
    297 void remove_all_memory_entries(struct mem_table_s *mtbl) 
     527static void remove_all_memory_entries(struct mem_table_s *mtbl) 
    298528{ 
    299529    /*  remove all ments, including their associated blocks */ 
     
    318548} 
    319549 
    320 void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file) 
     550static void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file) 
    321551{ 
    322552    /*  remove mtbl */ 
     
    339569 
    340570/*  evict the file @ index, must be less than FILE_TABLE_HASH_MAX   */ 
    341 void evict_file(unsigned int index) 
     571static void evict_file(unsigned int index) 
    342572{ 
    343573    if(DBG)fprintf(out, "evicting data @ index %d...\n", index); 
     
    510740} 
    511741 
    512 void update_lru(struct mem_table_s *mtbl, uint16_t index) 
     742static void update_lru(struct mem_table_s *mtbl, uint16_t index) 
    513743{ 
    514744            if(DBG)fprintf(out, "updating lru...\n"); 
     
    600830} 
    601831 
    602 void evict_LRU(struct mem_table_s *mtbl) 
     832static void evict_LRU(struct mem_table_s *mtbl) 
    603833{ 
    604834    if(DBG)fprintf(out, "evicting LRU...\n"); 
     
    767997} 
    768998 
    769 void print_lru(struct mem_table_s *mtbl) 
     999/*  The following two functions are provided for error checking purposes. 
     1000 */ 
     1001static void print_lru(struct mem_table_s *mtbl) 
    7701002{ 
    7711003    if(DBG)fprintf(out, "\tprinting lru list:\n"); 
     
    7791011} 
    7801012 
    781 void print_dirty(struct mem_table_s *mtbl) 
     1013static void print_dirty(struct mem_table_s *mtbl) 
    7821014{ 
    7831015    if(DBG)fprintf(out, "\tprinting dirty list:\n"); 
     
    7931025    } 
    7941026} 
     1027/*  End of Internal Only Functions    */ 
    7951028 
    7961029/* 
     
    8021035 * vim: ts=8 sts=4 sw=4 expandtab 
    8031036 */ 
    804  
    805  
  • branches/Orange-Branch/src/client/usrint/ucache.h

    r8977 r9001  
    55 */ 
    66 
    7 #ifndef UCACHE_H 
    8 #define UCACHE_H 1 
    9  
    10 #include <sys/types.h> 
     7#ifndef UCACHE_INTERNAL_H 
     8#define UCACHE_INTERNAL_H 
     9 
     10#include <stdio.h> 
    1111#include <sys/shm.h> 
    1212#include <stdint.h> 
     13#include <semaphore.h> 
     14 
     15/* The following includes may end up not being needed.  
     16#include <sys/types.h> 
     17#include <sys/sem.h> 
    1318#include <unistd.h> 
    14 #include <stdio.h> 
    1519#include <string.h> 
    1620#include <assert.h> 
     21*/ 
    1722 
    1823#define MEM_TABLE_ENTRY_COUNT 818 
     
    2732#define CACHE_SIZE (CACHE_BLOCK_SIZE_K * 1024 * BLOCKS_IN_CACHE) 
    2833#define AT_FLAGS 0 
    29 #define SVSHM_MODE   (SHM_R | SHM_W | SHM_R>>3 | SHM_R>>6)  
     34#define SVSHM_MODE (SHM_R | SHM_W | SHM_R>>3 | SHM_R>>6) 
    3035#define CACHE_FLAGS (SVSHM_MODE | IPC_CREAT) 
    31  
    3236#define NIL (-1) 
     37 
    3338#define DBG 0 
    34 #define F_EVICT 1   /*  Evict files if necessary    */ 
    35 #define M_EVICT 1   /*  Evict memory entries if necessary   */ 
     39#define INTERNAL_TESTING 0 
     40#define ucache_lock_t sem_t 
     41 
     42typedef uint32_t PVFS_fs_id; 
     43typedef uint64_t PVFS_object_ref; 
    3644 
    3745/** A link for one block of memory in a files hash table 
     
    99107    uint16_t free_mtbl_ent; /* entry index of next free mtbl */ 
    100108    uint16_t free_list; /* index of next free file entry */ 
     109    /*  pthread_spinlock_t spinlock */ 
    101110    char pad[12]; 
    102111    struct file_ent_s file[FILE_TABLE_ENTRY_COUNT]; 
     
    112121}; 
    113122 
    114 /*  Function Declarations   */ 
    115 static union user_cache_u *get_ucache(); 
     123/* externally visible API */ 
     124extern void ucache_initialize(void); 
     125extern int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 
     126extern void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 
     127extern void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 
     128extern int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 
     129extern int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 
     130extern int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 
     131 
     132/* Internal Only Function Declarations   */ 
     133    /*  Cache Locking Functions */ 
     134static int ucache_lock_init(ucache_lock_t * lock); 
     135static int ucache_lock_lock(ucache_lock_t * lock); 
     136static int ucache_lock_unlock(ucache_lock_t * lock); 
     137static int ucache_lock_getvalue(ucache_lock_t * lock, int *sval); 
     138static int ucache_lock_destroy(ucache_lock_t * lock); 
     139 
     140    /*  Dirty List Iterator */ 
     141static int dirty_done(uint16_t index); 
     142static int dirty_next(struct mem_table_s *mtbl, uint16_t index); 
     143 
     144    /*  Memory Entry Chain Iterator */ 
     145static int ment_done(int index); 
     146static int ment_next(struct mem_table_s *mtbl, int index); 
     147 
     148    /*  File Entry Chain Iterator   */ 
     149static int file_done(int index); 
     150static int file_next(struct file_table_s *ftbl, int index); 
     151 
    116152static void add_free_mtbls(int blk); 
    117 void ucache_initialize(void); 
    118153static void init_memory_table(int blk, int ent); 
    119154static uint16_t get_free_blk(void); 
     
    132167); 
    133168static int get_next_free_mtbl(uint32_t *free_mtbl_blk, uint16_t *free_mtbl_ent); 
    134 void remove_all_memory_entries(struct mem_table_s *mtbl); 
    135 void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file); 
    136 void evict_file(unsigned int index); 
     169static void remove_all_memory_entries(struct mem_table_s *mtbl); 
     170static void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file); 
     171static void evict_file(unsigned int index); 
    137172static struct mem_table_s *insert_file(uint32_t fs_id, uint64_t handle); 
    138173static int remove_file(uint32_t fs_id, uint64_t handle); 
     
    143178                    uint16_t *mem_ent_prev_index 
    144179); 
    145 void update_lru(struct mem_table_s *mtbl, uint16_t index); 
     180static void update_lru(struct mem_table_s *mtbl, uint16_t index); 
    146181static int locate_max_mtbl(struct mem_table_s **mtbl); 
    147 void evict_LRU(struct mem_table_s *mtbl); 
     182static void evict_LRU(struct mem_table_s *mtbl); 
    148183static void *set_item(struct mem_table_s *mtbl,  
    149184                    uint64_t offset,  
     
    152187static void *insert_mem(struct mem_table_s *mtbl, uint64_t offset); 
    153188static int remove_mem(struct mem_table_s *mtbl, uint64_t offset); 
    154  
    155 /*  Dirty List Iterator */ 
    156 static int dirty_done(uint16_t index) 
    157 { 
    158     return ((int16_t)index==NIL); 
    159 } 
    160 static int dirty_next(struct mem_table_s *mtbl, uint16_t index) 
    161 { 
    162     return mtbl->mem[index].dirty_next; 
    163 } 
    164  
    165 /*  Memory Entry Chain Iterator */ 
    166 static int ment_done(int index) 
    167 { 
    168     return ((int16_t)index==NIL); 
    169 } 
    170 static int ment_next(struct mem_table_s *mtbl, int index) 
    171 { 
    172     return mtbl->mem[index].next; 
    173 } 
    174  
    175 /*  File Entry Chain Iterator   */ 
    176 static int file_done(int index) 
    177 { 
    178     return ((int16_t)index==NIL); 
    179 } 
    180 static int file_next(struct file_table_s *ftbl, int index) 
    181 { 
    182     return ftbl->file[index].next; 
    183 } 
    184  
    185 /*  list printing functions */ 
    186 void print_lru(struct mem_table_s *mtbl); 
    187 void print_dirty(struct mem_table_s *mtbl); 
    188  
    189 /* externally visible API */ 
    190 #if 1 
    191 typedef uint32_t PVFS_fs_id; 
    192 typedef uint64_t PVFS_object_ref; 
    193 int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
    194 { 
    195     struct mem_table_s *mtbl= lookup_file( 
    196         (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
    197     if((int)mtbl==NIL) 
    198     { 
    199         insert_file((uint32_t)*fs_id, (uint64_t)*handle); 
    200     } 
    201     return 1; 
    202 } 
    203  
    204 void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
    205 { 
    206     struct mem_table_s *mtbl= lookup_file( 
    207         (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
    208     if((int)mtbl!=NIL) 
    209     { 
    210         return(lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, NULL));  
    211     } 
    212     return (void *)NIL; 
    213 } 
    214  
    215 void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
    216 { 
    217     struct mem_table_s *mtbl= lookup_file( 
    218         (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
    219     if((int)mtbl==NIL) 
    220     { 
    221         return (void *)NIL; 
    222     } 
    223     else 
    224     { 
    225         remove_mem(mtbl, (uint64_t)offset); 
    226         return insert_mem(mtbl, (uint64_t)offset);  
    227     } 
    228 } 
    229  
    230 int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 
    231 { 
    232     struct mem_table_s *mtbl= lookup_file( 
    233         (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
    234     if((int)mtbl!=NIL) 
    235     { 
    236         return remove_mem(mtbl, (uint64_t)offset);  
    237     } 
    238     return NIL; 
    239 } 
    240  
    241 int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
    242 { 
    243     struct mem_table_s *mtbl= lookup_file( 
    244         (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 
    245     if((int)mtbl==NIL) 
    246     { 
    247         return NIL; 
    248     } 
    249     int i; 
    250     for(i=mtbl->dirty_list; !dirty_done(i); i=dirty_next(mtbl, i)){ 
    251         struct mem_ent_s *ment = &(mtbl->mem[i]); 
    252         mtbl->mem[i].dirty_next = NIL; 
    253         if((int64_t)ment->tag==NIL || (int32_t)ment->item==NIL){ 
    254             break; 
    255         } 
    256         //flush block to disk 
    257     } 
    258     mtbl->dirty_list = NIL; 
    259     return 1; 
    260 } 
    261  
    262 int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 
    263 { 
    264     uint32_t file_mtbl_blk; 
    265     uint16_t file_mtbl_ent; 
    266     uint16_t file_ent_index; 
    267     uint16_t file_ent_prev_index; 
    268     struct mem_table_s *mtbl= lookup_file( 
    269         (uint32_t)(*fs_id),  
    270         (uint64_t)(*handle),  
    271         &file_mtbl_blk,  
    272         &file_mtbl_ent,  
    273         &file_ent_index,  
    274         &file_ent_prev_index); 
    275     if((int)mtbl==NIL) 
    276     { 
    277         return NIL; 
    278     } 
    279     remove_all_memory_entries(mtbl); 
    280     struct file_ent_s *file = &(get_ucache()->ftbl.file[file_ent_index]); 
    281     put_free_mtbl(mtbl, file); 
    282     put_free_fent(file_ent_index); 
    283 } 
     189    /*  list printing functions */ 
     190static void print_lru(struct mem_table_s *mtbl); 
     191static void print_dirty(struct mem_table_s *mtbl); 
     192/****************************************  End of Internal Only Functions    */ 
    284193#endif 
    285 #endif  
    286194 
    287195/*