Changeset 9001
- Timestamp:
- 08/09/11 13:04:46 (22 months ago)
- Location:
- branches/Orange-Branch/src/client/usrint
- Files:
-
- 2 modified
Legend:
- Unmodified
- Added
- Removed
-
branches/Orange-Branch/src/client/usrint/ucache.c
r8977 r9001 5 5 */ 6 6 7 /* Experimental cache for user data 8 * Currently under development. 9 */ 10 11 /* 12 * Note: When unsigned ints are set to NIL, their values are based on type: 13 * ex: 16 0xFFFF 14 * 32 0XFFFFFFFF 15 * 64 0XFFFFFFFFFFFFFFFF 16 * ALL EQUAL THE SIGNED REPRESENTATION OF -1, CALLED NIL. 17 */ 18 7 19 #include "ucache.h" 8 9 /*10 * Note: When unsigned ints are set to NIL, their values are based on type:11 * ex: 16 0xFFFF12 * 32 0XFFFFFFFF13 * 64 0XFFFFFFFFFFFFFFFF14 * ALL EQUAL THE SIGNED REPRESENTATION OF -1, CALLED NIL.15 */16 20 17 21 static union user_cache_u *ucache; 18 22 static int ucache_blk_cnt; 19 FILE *out; /* For Logging Purposes */ 20 21 static union user_cache_u *get_ucache(){ 22 return ucache; 23 } 24 25 /* This function should only be called when the ftbl has no free mtbls */ 26 static void add_free_mtbls(int blk) 27 { 28 int i, start_mtbl; 29 struct file_table_s *ftbl = &(ucache->ftbl); 30 union cache_block_u *b = &(ucache->b[blk]); 31 32 /* add mtbls in blk to ftbl free list */ 33 if (blk == 0) 34 { 35 start_mtbl = 1; /* skip blk 0 ent 0 which is ftbl */ 36 } 37 else 38 { 39 start_mtbl = 0; 40 } 41 for (i = start_mtbl; i < MTBL_PER_BLOCK - 1; i++) 42 { 43 b->mtbl[i].free_list_blk = blk; 44 b->mtbl[i].free_list = i + 1; 45 } 46 b->mtbl[i].free_list_blk = NIL; 47 b->mtbl[i].free_list = NIL; 48 ftbl->free_mtbl_blk = blk; 49 ftbl->free_mtbl_ent = start_mtbl; 50 } 51 52 void ucache_initialize(void) 53 { 23 static ucache_lock_t *ucache_lock; /* For maintaining concurrency */ 24 static FILE *out; /* For Logging Purposes */ 25 26 /* Externally Visible API 27 * The following functions are thread/processor safe regarding the cache 28 * tables and data. 29 */ 30 31 /* Initializes the cache. 32 * Mainly, it aquires a shared memory segment used to cache data. 33 * 34 * This function also initializes the the FTBL and some MTBLs. 35 * 36 * The whole cache is protected by a locking mechanism to maintain concurrency. 37 * Currently using posix semaphores 38 */ 39 extern void ucache_initialize(void) 40 { 41 /* Aquire shared memory for ucache_lock */ 42 ucache_lock = shmat(shmget(ftok(GET_KEY_FILE, 'a'), 43 sizeof(ucache_lock_t), CACHE_FLAGS), NULL, AT_FLAGS); 44 ucache_lock_init(ucache_lock); 45 ucache_lock_lock(ucache_lock); 46 /* Aquire shared memory for lock for ucache */ 54 47 int key, id, i; 55 48 char *key_file_path; … … 96 89 } 97 90 ucache->ftbl.file[FILE_TABLE_ENTRY_COUNT - 1].next = NIL; 91 ucache_lock_unlock(ucache_lock); 92 } 93 94 extern int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 95 { 96 ucache_lock_lock(ucache_lock); 97 struct mem_table_s *mtbl= lookup_file( 98 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 99 if((int)mtbl==NIL) 100 { 101 insert_file((uint32_t)*fs_id, (uint64_t)*handle); 102 } 103 ucache_lock_unlock(ucache_lock); 104 return 1; 105 } 106 107 /* Returns ptr to block in cache based on file and offset */ 108 extern void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 109 { 110 ucache_lock_lock(ucache_lock); 111 struct mem_table_s *mtbl= lookup_file( 112 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 113 if((int)mtbl!=NIL) 114 { 115 char *retVal = (char *)lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, NULL); 116 ucache_lock_unlock(ucache_lock); 117 return((void *)retVal); 118 } 119 ucache_lock_unlock(ucache_lock); 120 return (void *)NIL; 121 } 122 123 /* Inserts block of data into cache */ 124 extern void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 125 { 126 ucache_lock_lock(ucache_lock); 127 struct mem_table_s *mtbl= lookup_file( 128 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 129 if((int)mtbl==NIL) 130 { 131 ucache_lock_unlock(ucache_lock); 132 return (void *)NIL; 133 } 134 else 135 { 136 remove_mem(mtbl, (uint64_t)offset); 137 char * retVal= insert_mem(mtbl, (uint64_t)offset); 138 ucache_lock_unlock(ucache_lock); 139 return ((void *)retVal); 140 } 141 } 142 143 /* Removes a cached block of data from mtbl */ 144 extern int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 145 { 146 ucache_lock_lock(ucache_lock); 147 struct mem_table_s *mtbl= lookup_file( 148 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 149 if((int)mtbl!=NIL) 150 { 151 int retVal = remove_mem(mtbl, (uint64_t)offset); 152 ucache_lock_unlock(ucache_lock); 153 return retVal; 154 } 155 ucache_lock_unlock(ucache_lock); 156 return NIL; 157 } 158 159 /* Flushes dirty blocks to the I/O Nodes */ 160 extern int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 161 { 162 ucache_lock_lock(ucache_lock); 163 struct mem_table_s *mtbl= lookup_file( 164 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 165 if((int)mtbl==NIL) 166 { 167 return NIL; 168 } 169 int i; 170 for(i=mtbl->dirty_list; !dirty_done(i); i=dirty_next(mtbl, i)){ 171 struct mem_ent_s *ment = &(mtbl->mem[i]); 172 mtbl->mem[i].dirty_next = NIL; 173 if((int64_t)ment->tag==NIL || (int32_t)ment->item==NIL){ 174 break; 175 } 176 /* //flush block to disk */ 177 } 178 mtbl->dirty_list = NIL; 179 ucache_lock_unlock(ucache_lock); 180 return 1; 181 } 182 183 /* Removes all memory entries in the mtbl corresponding to the file info 184 * provided as parameters. It also removes the mtbl and the file entry from 185 * the cache. 186 */ 187 extern int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 188 { 189 ucache_lock_lock(ucache_lock); 190 uint32_t file_mtbl_blk; 191 uint16_t file_mtbl_ent; 192 uint16_t file_ent_index; 193 uint16_t file_ent_prev_index; 194 struct mem_table_s *mtbl= lookup_file( 195 (uint32_t)(*fs_id), 196 (uint64_t)(*handle), 197 &file_mtbl_blk, 198 &file_mtbl_ent, 199 &file_ent_index, 200 &file_ent_prev_index); 201 if((int)mtbl==NIL) 202 { 203 ucache_lock_unlock(ucache_lock); 204 return NIL; 205 } 206 remove_all_memory_entries(mtbl); 207 struct file_ent_s *file = &(ucache->ftbl.file[file_ent_index]); 208 put_free_mtbl(mtbl, file); 209 put_free_fent(file_ent_index); 210 ucache_lock_unlock(ucache_lock); 211 return 1; 212 } 213 214 /* Use the following function to decrement the reference count of a particular 215 * mtbl. This function must be called when a user's code is done with the 216 * pointer provided via any of the pointer returning external functions listed 217 * above. 218 */ 219 extern int ucache_dec_ref_cnt(struct mem_table_s * mtbl) 220 { 221 ucache_lock_lock(ucache_lock); 222 /* decrement ref_cnt of mtbl */ 223 mtbl->ref_cnt--; 224 ucache_lock_unlock(ucache_lock); 225 } 226 /***************************************** End of Externally Visible API */ 227 228 229 /* Beginning of internal only (static) functions 230 */ 231 232 /* Internally Available Locking Mechanism - using POSIX semaphores 233 */ 234 static int ucache_lock_init(ucache_lock_t * lock) 235 { 236 /* Set pshared (2nd arg) to non-zero value to share semaphore b/w forked 237 * processes 238 */ 239 return sem_init(lock, 1, 1); 240 } 241 242 /* Returns 0 when lock is locked; otherwise, return -1 and sets errno*/ 243 static int ucache_lock_lock(ucache_lock_t * lock) 244 { 245 return sem_wait(lock); 246 } 247 248 /* If successful, return zero; otherwise, return -1 and sets errno */ 249 static int ucache_lock_unlock(ucache_lock_t * lock) 250 { 251 return sem_post(lock); 252 } 253 254 /* Upon successful completion, returns zero; otherwise, returns and sets errno. 255 */ 256 static int ucache_lock_getvalue(ucache_lock_t * lock, int *sval) 257 { 258 return sem_getvalue(lock, sval); 259 } 260 261 /* Upon successful completion, returns zero; otherwise returns 1 and sets errno. 262 */ 263 static int ucache_lock_destroy(ucache_lock_t * lock) 264 { 265 return sem_destroy(lock); 266 } 267 268 /* Dirty List Iterator */ 269 static int dirty_done(uint16_t index) 270 { 271 return ((int16_t)index==NIL); 272 } 273 274 static int dirty_next(struct mem_table_s *mtbl, uint16_t index) 275 { 276 return mtbl->mem[index].dirty_next; 277 } 278 279 /* Memory Entry Chain Iterator */ 280 static int ment_done(int index) 281 { 282 return ((int16_t)index==NIL); 283 } 284 285 static int ment_next(struct mem_table_s *mtbl, int index) 286 { 287 return mtbl->mem[index].next; 288 } 289 290 /* File Entry Chain Iterator */ 291 static int file_done(int index) 292 { 293 return ((int16_t)index==NIL); 294 } 295 296 static int file_next(struct file_table_s *ftbl, int index) 297 { 298 return ftbl->file[index].next; 299 } 300 301 /* 302 * This function should only be called when the ftbl has no free mtbls. 303 */ 304 static void add_free_mtbls(int blk) 305 { 306 int i, start_mtbl; 307 struct file_table_s *ftbl = &(ucache->ftbl); 308 union cache_block_u *b = &(ucache->b[blk]); 309 310 /* add mtbls in blk to ftbl free list */ 311 if (blk == 0) 312 { 313 start_mtbl = 1; /* skip blk 0 ent 0 which is ftbl */ 314 } 315 else 316 { 317 start_mtbl = 0; 318 } 319 for (i = start_mtbl; i < MTBL_PER_BLOCK - 1; i++) 320 { 321 b->mtbl[i].free_list_blk = blk; 322 b->mtbl[i].free_list = i + 1; 323 } 324 b->mtbl[i].free_list_blk = NIL; 325 b->mtbl[i].free_list = NIL; 326 ftbl->free_mtbl_blk = blk; 327 ftbl->free_mtbl_ent = start_mtbl; 98 328 } 99 329 … … 228 458 struct file_table_s *ftbl = &(ucache->ftbl); 229 459 struct file_ent_s *current; /* Current ptr for iteration */ 230 int index;/* Index into file hash table */231 in dex = handle % FILE_TABLE_HASH_MAX;460 /* Index into file hash table */ 461 int index = handle % FILE_TABLE_HASH_MAX; 232 462 if(DBG)fprintf(out, "\thashed index: %d\n", index); 233 463 current = &(ftbl->file[index]); … … 295 525 } 296 526 297 void remove_all_memory_entries(struct mem_table_s *mtbl)527 static void remove_all_memory_entries(struct mem_table_s *mtbl) 298 528 { 299 529 /* remove all ments, including their associated blocks */ … … 318 548 } 319 549 320 void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file)550 static void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file) 321 551 { 322 552 /* remove mtbl */ … … 339 569 340 570 /* evict the file @ index, must be less than FILE_TABLE_HASH_MAX */ 341 void evict_file(unsigned int index)571 static void evict_file(unsigned int index) 342 572 { 343 573 if(DBG)fprintf(out, "evicting data @ index %d...\n", index); … … 510 740 } 511 741 512 void update_lru(struct mem_table_s *mtbl, uint16_t index)742 static void update_lru(struct mem_table_s *mtbl, uint16_t index) 513 743 { 514 744 if(DBG)fprintf(out, "updating lru...\n"); … … 600 830 } 601 831 602 void evict_LRU(struct mem_table_s *mtbl)832 static void evict_LRU(struct mem_table_s *mtbl) 603 833 { 604 834 if(DBG)fprintf(out, "evicting LRU...\n"); … … 767 997 } 768 998 769 void print_lru(struct mem_table_s *mtbl) 999 /* The following two functions are provided for error checking purposes. 1000 */ 1001 static void print_lru(struct mem_table_s *mtbl) 770 1002 { 771 1003 if(DBG)fprintf(out, "\tprinting lru list:\n"); … … 779 1011 } 780 1012 781 void print_dirty(struct mem_table_s *mtbl)1013 static void print_dirty(struct mem_table_s *mtbl) 782 1014 { 783 1015 if(DBG)fprintf(out, "\tprinting dirty list:\n"); … … 793 1025 } 794 1026 } 1027 /* End of Internal Only Functions */ 795 1028 796 1029 /* … … 802 1035 * vim: ts=8 sts=4 sw=4 expandtab 803 1036 */ 804 805 -
branches/Orange-Branch/src/client/usrint/ucache.h
r8977 r9001 5 5 */ 6 6 7 #ifndef UCACHE_ H8 #define UCACHE_ H 19 10 #include <s ys/types.h>7 #ifndef UCACHE_INTERNAL_H 8 #define UCACHE_INTERNAL_H 9 10 #include <stdio.h> 11 11 #include <sys/shm.h> 12 12 #include <stdint.h> 13 #include <semaphore.h> 14 15 /* The following includes may end up not being needed. 16 #include <sys/types.h> 17 #include <sys/sem.h> 13 18 #include <unistd.h> 14 #include <stdio.h>15 19 #include <string.h> 16 20 #include <assert.h> 21 */ 17 22 18 23 #define MEM_TABLE_ENTRY_COUNT 818 … … 27 32 #define CACHE_SIZE (CACHE_BLOCK_SIZE_K * 1024 * BLOCKS_IN_CACHE) 28 33 #define AT_FLAGS 0 29 #define SVSHM_MODE (SHM_R | SHM_W | SHM_R>>3 | SHM_R>>6)34 #define SVSHM_MODE (SHM_R | SHM_W | SHM_R>>3 | SHM_R>>6) 30 35 #define CACHE_FLAGS (SVSHM_MODE | IPC_CREAT) 31 32 36 #define NIL (-1) 37 33 38 #define DBG 0 34 #define F_EVICT 1 /* Evict files if necessary */ 35 #define M_EVICT 1 /* Evict memory entries if necessary */ 39 #define INTERNAL_TESTING 0 40 #define ucache_lock_t sem_t 41 42 typedef uint32_t PVFS_fs_id; 43 typedef uint64_t PVFS_object_ref; 36 44 37 45 /** A link for one block of memory in a files hash table … … 99 107 uint16_t free_mtbl_ent; /* entry index of next free mtbl */ 100 108 uint16_t free_list; /* index of next free file entry */ 109 /* pthread_spinlock_t spinlock */ 101 110 char pad[12]; 102 111 struct file_ent_s file[FILE_TABLE_ENTRY_COUNT]; … … 112 121 }; 113 122 114 /* Function Declarations */ 115 static union user_cache_u *get_ucache(); 123 /* externally visible API */ 124 extern void ucache_initialize(void); 125 extern int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 126 extern void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 127 extern void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 128 extern int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset); 129 extern int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 130 extern int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle); 131 132 /* Internal Only Function Declarations */ 133 /* Cache Locking Functions */ 134 static int ucache_lock_init(ucache_lock_t * lock); 135 static int ucache_lock_lock(ucache_lock_t * lock); 136 static int ucache_lock_unlock(ucache_lock_t * lock); 137 static int ucache_lock_getvalue(ucache_lock_t * lock, int *sval); 138 static int ucache_lock_destroy(ucache_lock_t * lock); 139 140 /* Dirty List Iterator */ 141 static int dirty_done(uint16_t index); 142 static int dirty_next(struct mem_table_s *mtbl, uint16_t index); 143 144 /* Memory Entry Chain Iterator */ 145 static int ment_done(int index); 146 static int ment_next(struct mem_table_s *mtbl, int index); 147 148 /* File Entry Chain Iterator */ 149 static int file_done(int index); 150 static int file_next(struct file_table_s *ftbl, int index); 151 116 152 static void add_free_mtbls(int blk); 117 void ucache_initialize(void);118 153 static void init_memory_table(int blk, int ent); 119 154 static uint16_t get_free_blk(void); … … 132 167 ); 133 168 static int get_next_free_mtbl(uint32_t *free_mtbl_blk, uint16_t *free_mtbl_ent); 134 void remove_all_memory_entries(struct mem_table_s *mtbl);135 void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file);136 void evict_file(unsigned int index);169 static void remove_all_memory_entries(struct mem_table_s *mtbl); 170 static void put_free_mtbl(struct mem_table_s *mtbl, struct file_ent_s *file); 171 static void evict_file(unsigned int index); 137 172 static struct mem_table_s *insert_file(uint32_t fs_id, uint64_t handle); 138 173 static int remove_file(uint32_t fs_id, uint64_t handle); … … 143 178 uint16_t *mem_ent_prev_index 144 179 ); 145 void update_lru(struct mem_table_s *mtbl, uint16_t index);180 static void update_lru(struct mem_table_s *mtbl, uint16_t index); 146 181 static int locate_max_mtbl(struct mem_table_s **mtbl); 147 void evict_LRU(struct mem_table_s *mtbl);182 static void evict_LRU(struct mem_table_s *mtbl); 148 183 static void *set_item(struct mem_table_s *mtbl, 149 184 uint64_t offset, … … 152 187 static void *insert_mem(struct mem_table_s *mtbl, uint64_t offset); 153 188 static int remove_mem(struct mem_table_s *mtbl, uint64_t offset); 154 155 /* Dirty List Iterator */ 156 static int dirty_done(uint16_t index) 157 { 158 return ((int16_t)index==NIL); 159 } 160 static int dirty_next(struct mem_table_s *mtbl, uint16_t index) 161 { 162 return mtbl->mem[index].dirty_next; 163 } 164 165 /* Memory Entry Chain Iterator */ 166 static int ment_done(int index) 167 { 168 return ((int16_t)index==NIL); 169 } 170 static int ment_next(struct mem_table_s *mtbl, int index) 171 { 172 return mtbl->mem[index].next; 173 } 174 175 /* File Entry Chain Iterator */ 176 static int file_done(int index) 177 { 178 return ((int16_t)index==NIL); 179 } 180 static int file_next(struct file_table_s *ftbl, int index) 181 { 182 return ftbl->file[index].next; 183 } 184 185 /* list printing functions */ 186 void print_lru(struct mem_table_s *mtbl); 187 void print_dirty(struct mem_table_s *mtbl); 188 189 /* externally visible API */ 190 #if 1 191 typedef uint32_t PVFS_fs_id; 192 typedef uint64_t PVFS_object_ref; 193 int ucache_open_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 194 { 195 struct mem_table_s *mtbl= lookup_file( 196 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 197 if((int)mtbl==NIL) 198 { 199 insert_file((uint32_t)*fs_id, (uint64_t)*handle); 200 } 201 return 1; 202 } 203 204 void *ucache_lookup(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 205 { 206 struct mem_table_s *mtbl= lookup_file( 207 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 208 if((int)mtbl!=NIL) 209 { 210 return(lookup_mem(mtbl, (uint64_t)offset, NULL, NULL, NULL)); 211 } 212 return (void *)NIL; 213 } 214 215 void *ucache_insert(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 216 { 217 struct mem_table_s *mtbl= lookup_file( 218 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 219 if((int)mtbl==NIL) 220 { 221 return (void *)NIL; 222 } 223 else 224 { 225 remove_mem(mtbl, (uint64_t)offset); 226 return insert_mem(mtbl, (uint64_t)offset); 227 } 228 } 229 230 int ucache_remove(PVFS_fs_id *fs_id, PVFS_object_ref *handle, uint64_t offset) 231 { 232 struct mem_table_s *mtbl= lookup_file( 233 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 234 if((int)mtbl!=NIL) 235 { 236 return remove_mem(mtbl, (uint64_t)offset); 237 } 238 return NIL; 239 } 240 241 int ucache_flush(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 242 { 243 struct mem_table_s *mtbl= lookup_file( 244 (uint32_t)(*fs_id), (uint64_t)(*handle), NULL, NULL, NULL, NULL); 245 if((int)mtbl==NIL) 246 { 247 return NIL; 248 } 249 int i; 250 for(i=mtbl->dirty_list; !dirty_done(i); i=dirty_next(mtbl, i)){ 251 struct mem_ent_s *ment = &(mtbl->mem[i]); 252 mtbl->mem[i].dirty_next = NIL; 253 if((int64_t)ment->tag==NIL || (int32_t)ment->item==NIL){ 254 break; 255 } 256 //flush block to disk 257 } 258 mtbl->dirty_list = NIL; 259 return 1; 260 } 261 262 int ucache_close_file(PVFS_fs_id *fs_id, PVFS_object_ref *handle) 263 { 264 uint32_t file_mtbl_blk; 265 uint16_t file_mtbl_ent; 266 uint16_t file_ent_index; 267 uint16_t file_ent_prev_index; 268 struct mem_table_s *mtbl= lookup_file( 269 (uint32_t)(*fs_id), 270 (uint64_t)(*handle), 271 &file_mtbl_blk, 272 &file_mtbl_ent, 273 &file_ent_index, 274 &file_ent_prev_index); 275 if((int)mtbl==NIL) 276 { 277 return NIL; 278 } 279 remove_all_memory_entries(mtbl); 280 struct file_ent_s *file = &(get_ucache()->ftbl.file[file_ent_index]); 281 put_free_mtbl(mtbl, file); 282 put_free_fent(file_ent_index); 283 } 189 /* list printing functions */ 190 static void print_lru(struct mem_table_s *mtbl); 191 static void print_dirty(struct mem_table_s *mtbl); 192 /**************************************** End of Internal Only Functions */ 284 193 #endif 285 #endif286 194 287 195 /*
