| 1 | /* |
|---|
| 2 | * (C) 2003 Clemson University and The University of Chicago |
|---|
| 3 | * |
|---|
| 4 | * See COPYING in top-level directory. |
|---|
| 5 | */ |
|---|
| 6 | |
|---|
| 7 | /** \file |
|---|
| 8 | * Declarations for state machine processing on clients. |
|---|
| 9 | */ |
|---|
| 10 | |
|---|
| 11 | #ifndef __PVFS2_CLIENT_STATE_MACHINE_H |
|---|
| 12 | #define __PVFS2_CLIENT_STATE_MACHINE_H |
|---|
| 13 | |
|---|
| 14 | /* |
|---|
| 15 | NOTE: state-machine.h is included at the bottom so we can define all |
|---|
| 16 | the client-sm structures before it's included |
|---|
| 17 | */ |
|---|
| 18 | #include "pvfs2-sysint.h" |
|---|
| 19 | #include "pvfs2-types.h" |
|---|
| 20 | #include "pvfs2-storage.h" |
|---|
| 21 | #include "pvfs2-util.h" |
|---|
| 22 | #include "PINT-reqproto-encode.h" |
|---|
| 23 | #include "job.h" |
|---|
| 24 | #include "trove.h" |
|---|
| 25 | #include "acache.h" |
|---|
| 26 | #include "id-generator.h" |
|---|
| 27 | #include "msgpairarray.h" |
|---|
| 28 | #include "pint-sysint-utils.h" |
|---|
| 29 | #include "pint-perf-counter.h" |
|---|
| 30 | #include "state-machine.h" |
|---|
| 31 | #include "pvfs2-hint.h" |
|---|
| 32 | #include "pint-event.h" |
|---|
| 33 | |
|---|
| 34 | #define MAX_LOOKUP_SEGMENTS PVFS_REQ_LIMIT_PATH_SEGMENT_COUNT |
|---|
| 35 | #define MAX_LOOKUP_CONTEXTS PVFS_REQ_LIMIT_MAX_SYMLINK_RESOLUTION_COUNT |
|---|
| 36 | |
|---|
| 37 | /* Default client timeout in seconds used to set the timeout for jobs that |
|---|
| 38 | * send or receive request messages. |
|---|
| 39 | */ |
|---|
| 40 | #ifndef PVFS2_CLIENT_JOB_BMI_TIMEOUT_DEFAULT |
|---|
| 41 | #define PVFS2_CLIENT_JOB_BMI_TIMEOUT_DEFAULT 30 |
|---|
| 42 | #endif |
|---|
| 43 | |
|---|
| 44 | /* Default number of times to retry restartable client operations. */ |
|---|
| 45 | #define PVFS2_CLIENT_RETRY_LIMIT_DEFAULT (5) |
|---|
| 46 | |
|---|
| 47 | /* Default number of milliseconds to delay before retries */ |
|---|
| 48 | #define PVFS2_CLIENT_RETRY_DELAY_MS_DEFAULT 2000 |
|---|
| 49 | |
|---|
| 50 | int PINT_client_state_machine_initialize(void); |
|---|
| 51 | void PINT_client_state_machine_finalize(void); |
|---|
| 52 | job_context_id PINT_client_get_sm_context(void); |
|---|
| 53 | |
|---|
| 54 | /* PINT_client_sm_recv_state_s |
|---|
| 55 | * |
|---|
| 56 | * This is used for extra receives, such as acknowledgements from |
|---|
| 57 | * servers at the end of write operations. |
|---|
| 58 | */ |
|---|
| 59 | typedef struct PINT_client_sm_recv_state_s |
|---|
| 60 | { |
|---|
| 61 | int max_resp_sz; |
|---|
| 62 | void *encoded_resp_p; |
|---|
| 63 | job_id_t recv_id; |
|---|
| 64 | job_status_s recv_status; |
|---|
| 65 | PVFS_error op_status; |
|---|
| 66 | } PINT_client_sm_recv_state; |
|---|
| 67 | |
|---|
| 68 | struct PINT_client_remove_sm |
|---|
| 69 | { |
|---|
| 70 | char *object_name; /* input parameter */ |
|---|
| 71 | int stored_error_code; |
|---|
| 72 | int retry_count; |
|---|
| 73 | }; |
|---|
| 74 | |
|---|
| 75 | struct PINT_client_create_sm |
|---|
| 76 | { |
|---|
| 77 | char *object_name; /* input parameter */ |
|---|
| 78 | PVFS_object_attr attr; /* input parameter */ |
|---|
| 79 | PVFS_sysresp_create *create_resp; /* in/out parameter */ |
|---|
| 80 | |
|---|
| 81 | int retry_count; |
|---|
| 82 | int num_data_files; |
|---|
| 83 | int user_requested_num_data_files; |
|---|
| 84 | int stored_error_code; |
|---|
| 85 | |
|---|
| 86 | PINT_dist *dist; |
|---|
| 87 | PVFS_sys_layout layout; |
|---|
| 88 | |
|---|
| 89 | PVFS_handle metafile_handle; |
|---|
| 90 | int datafile_count; |
|---|
| 91 | PVFS_handle *datafile_handles; |
|---|
| 92 | int stuffed; |
|---|
| 93 | |
|---|
| 94 | int dirent_file_count; |
|---|
| 95 | PVFS_handle *dirent_handle; |
|---|
| 96 | |
|---|
| 97 | PVFS_handle handles[2]; |
|---|
| 98 | }; |
|---|
| 99 | |
|---|
| 100 | struct PINT_client_mkdir_sm |
|---|
| 101 | { |
|---|
| 102 | char *object_name; /* input parameter */ |
|---|
| 103 | PVFS_sysresp_mkdir *mkdir_resp; /* in/out parameter */ |
|---|
| 104 | PVFS_sys_attr sys_attr; /* input parameter */ |
|---|
| 105 | PVFS_ds_keyval *key_array; |
|---|
| 106 | PVFS_ds_keyval *val_array; |
|---|
| 107 | |
|---|
| 108 | int retry_count; |
|---|
| 109 | int stored_error_code; |
|---|
| 110 | PVFS_handle metafile_handle; |
|---|
| 111 | |
|---|
| 112 | /* keep first */ |
|---|
| 113 | PINT_dist *dist; |
|---|
| 114 | PVFS_sys_layout layout; |
|---|
| 115 | int num_dirent_files; |
|---|
| 116 | }; |
|---|
| 117 | |
|---|
| 118 | struct PINT_client_symlink_sm |
|---|
| 119 | { |
|---|
| 120 | char *link_name; /* input parameter */ |
|---|
| 121 | char *link_target; /* input parameter */ |
|---|
| 122 | PVFS_sysresp_symlink *sym_resp; /* in/out parameter*/ |
|---|
| 123 | PVFS_sys_attr sys_attr; /* input parameter */ |
|---|
| 124 | |
|---|
| 125 | int retry_count; |
|---|
| 126 | int stored_error_code; |
|---|
| 127 | PVFS_handle symlink_handle; |
|---|
| 128 | }; |
|---|
| 129 | |
|---|
| 130 | struct PINT_client_getattr_sm |
|---|
| 131 | { |
|---|
| 132 | PVFS_sysresp_getattr *getattr_resp_p; /* destination for output */ |
|---|
| 133 | }; |
|---|
| 134 | |
|---|
| 135 | struct PINT_client_setattr_sm |
|---|
| 136 | { |
|---|
| 137 | PVFS_sys_attr sys_attr; /* input parameter */ |
|---|
| 138 | }; |
|---|
| 139 | |
|---|
| 140 | struct PINT_client_mgmt_remove_dirent_sm |
|---|
| 141 | { |
|---|
| 142 | char *entry; |
|---|
| 143 | }; |
|---|
| 144 | |
|---|
| 145 | struct PINT_client_mgmt_create_dirent_sm |
|---|
| 146 | { |
|---|
| 147 | char *entry; |
|---|
| 148 | PVFS_handle entry_handle; |
|---|
| 149 | }; |
|---|
| 150 | |
|---|
| 151 | struct PINT_client_mgmt_get_dirdata_handle_sm |
|---|
| 152 | { |
|---|
| 153 | PVFS_handle *dirdata_handle; |
|---|
| 154 | }; |
|---|
| 155 | |
|---|
| 156 | /* this structure is used to handle mirrored retries in the small-io case*/ |
|---|
| 157 | typedef struct PINT_client_mirror_ctx |
|---|
| 158 | { |
|---|
| 159 | /*which copy of the mirrored handle are we using?*/ |
|---|
| 160 | uint32_t current_copies_count; |
|---|
| 161 | |
|---|
| 162 | /*the primary datahandle*/ |
|---|
| 163 | PVFS_handle original_datahandle; |
|---|
| 164 | |
|---|
| 165 | /*the server_nr for the primary datahandle*/ |
|---|
| 166 | uint32_t original_server_nr; |
|---|
| 167 | |
|---|
| 168 | /*do we retry the primary or use a mirrored handle?*/ |
|---|
| 169 | PVFS_boolean retry_original; |
|---|
| 170 | |
|---|
| 171 | /*did the current message for this handle complete without any errors?*/ |
|---|
| 172 | PVFS_boolean msg_completed; |
|---|
| 173 | |
|---|
| 174 | } PINT_client_small_io_ctx; |
|---|
| 175 | |
|---|
| 176 | |
|---|
| 177 | |
|---|
| 178 | /* this structure is used to handle mirrored retries when |
|---|
| 179 | * pvfs2_client_datafile_getattr_sizes_sm is called. |
|---|
| 180 | */ |
|---|
| 181 | typedef struct PINT_client_mirror_ctx PINT_client_getattr_mirror_ctx; |
|---|
| 182 | |
|---|
| 183 | |
|---|
| 184 | |
|---|
| 185 | typedef struct PINT_client_io_ctx |
|---|
| 186 | { |
|---|
| 187 | /* the index of the current context (in the context array) */ |
|---|
| 188 | int index; |
|---|
| 189 | |
|---|
| 190 | /* the metafile's dfile server index we're communicating with */ |
|---|
| 191 | int server_nr; |
|---|
| 192 | |
|---|
| 193 | /* the data handle we're responsible for doing I/O on */ |
|---|
| 194 | PVFS_handle data_handle; |
|---|
| 195 | |
|---|
| 196 | /* first level index into mirror_dfile_array. second level is */ |
|---|
| 197 | /* the server_nr. mirror_dfile_array[current_copies_count][server_nr] */ |
|---|
| 198 | uint32_t current_copies_count; |
|---|
| 199 | |
|---|
| 200 | /* increment after one set of mirrors have been tried. */ |
|---|
| 201 | uint32_t local_retry_count; |
|---|
| 202 | |
|---|
| 203 | /* should we retry the original or not? */ |
|---|
| 204 | uint32_t retry_original; |
|---|
| 205 | |
|---|
| 206 | job_id_t flow_job_id; |
|---|
| 207 | job_status_s flow_status; |
|---|
| 208 | flow_descriptor flow_desc; |
|---|
| 209 | PVFS_msg_tag_t session_tag; |
|---|
| 210 | |
|---|
| 211 | PINT_sm_msgpair_state msg; |
|---|
| 212 | PINT_client_sm_recv_state write_ack; |
|---|
| 213 | |
|---|
| 214 | /* |
|---|
| 215 | all *_has_been_posted fields are used at io_analyze_results time |
|---|
| 216 | to know if we should be checking for errors on particular fields |
|---|
| 217 | */ |
|---|
| 218 | int msg_send_has_been_posted; |
|---|
| 219 | int msg_recv_has_been_posted; |
|---|
| 220 | int flow_has_been_posted; |
|---|
| 221 | int write_ack_has_been_posted; |
|---|
| 222 | |
|---|
| 223 | /* |
|---|
| 224 | all *_in_progress fields are used at cancellation time to |
|---|
| 225 | determine what operations are currently in flight |
|---|
| 226 | */ |
|---|
| 227 | int msg_send_in_progress; |
|---|
| 228 | int msg_recv_in_progress; |
|---|
| 229 | int flow_in_progress; |
|---|
| 230 | int write_ack_in_progress; |
|---|
| 231 | |
|---|
| 232 | } PINT_client_io_ctx; |
|---|
| 233 | |
|---|
| 234 | struct PINT_client_io_sm |
|---|
| 235 | { |
|---|
| 236 | /* input parameters */ |
|---|
| 237 | enum PVFS_io_type io_type; |
|---|
| 238 | PVFS_Request file_req; |
|---|
| 239 | PVFS_offset file_req_offset; |
|---|
| 240 | void *buffer; |
|---|
| 241 | PVFS_Request mem_req; |
|---|
| 242 | |
|---|
| 243 | /* output parameter */ |
|---|
| 244 | PVFS_sysresp_io *io_resp_p; |
|---|
| 245 | |
|---|
| 246 | enum PVFS_flowproto_type flowproto_type; |
|---|
| 247 | enum PVFS_encoding_type encoding; |
|---|
| 248 | |
|---|
| 249 | int *datafile_index_array; |
|---|
| 250 | int datafile_count; |
|---|
| 251 | |
|---|
| 252 | int msgpair_completion_count; |
|---|
| 253 | int flow_completion_count; |
|---|
| 254 | int write_ack_completion_count; |
|---|
| 255 | |
|---|
| 256 | PINT_client_io_ctx *contexts; |
|---|
| 257 | int context_count; |
|---|
| 258 | |
|---|
| 259 | PINT_client_small_io_ctx *small_io_ctx; |
|---|
| 260 | |
|---|
| 261 | int total_cancellations_remaining; |
|---|
| 262 | |
|---|
| 263 | int retry_count; |
|---|
| 264 | int stored_error_code; |
|---|
| 265 | |
|---|
| 266 | PVFS_size total_size; |
|---|
| 267 | |
|---|
| 268 | PVFS_size * dfile_size_array; |
|---|
| 269 | int small_io; |
|---|
| 270 | }; |
|---|
| 271 | |
|---|
| 272 | struct PINT_client_flush_sm |
|---|
| 273 | { |
|---|
| 274 | }; |
|---|
| 275 | |
|---|
| 276 | struct PINT_client_readdir_sm |
|---|
| 277 | { |
|---|
| 278 | PVFS_ds_position pos_token; /* in/out parameter */ |
|---|
| 279 | int dirent_limit; /* input parameter */ |
|---|
| 280 | int dirdata_index; /* in/out parameter */ |
|---|
| 281 | PVFS_sysresp_readdir *readdir_resp; /* in/out parameter*/ |
|---|
| 282 | |
|---|
| 283 | int num_dirdata_needed; /* tmp parameter */ |
|---|
| 284 | }; |
|---|
| 285 | |
|---|
| 286 | struct handle_to_index { |
|---|
| 287 | PVFS_handle handle; |
|---|
| 288 | int handle_index;/* This is the index into the dirent array itself */ |
|---|
| 289 | int aux_index; /* this is used to store the ordinality of the dfile handles */ |
|---|
| 290 | }; |
|---|
| 291 | |
|---|
| 292 | struct PINT_client_readdirplus_sm |
|---|
| 293 | { |
|---|
| 294 | PVFS_ds_position pos_token; /* input parameter */ |
|---|
| 295 | int dirent_limit; /* input parameter */ |
|---|
| 296 | int attrmask; /* input parameter */ |
|---|
| 297 | PVFS_sysresp_readdirplus *readdirplus_resp; /* in/out parameter*/ |
|---|
| 298 | /* scratch variables */ |
|---|
| 299 | int nhandles; |
|---|
| 300 | int svr_count; |
|---|
| 301 | PVFS_size **size_array; |
|---|
| 302 | PVFS_object_attr *obj_attr_array; |
|---|
| 303 | struct handle_to_index *input_handle_array; |
|---|
| 304 | PVFS_BMI_addr_t *server_addresses; |
|---|
| 305 | int *handle_count; |
|---|
| 306 | PVFS_handle **handles; |
|---|
| 307 | }; |
|---|
| 308 | |
|---|
| 309 | typedef struct |
|---|
| 310 | { |
|---|
| 311 | char *seg_name; |
|---|
| 312 | char *seg_remaining; |
|---|
| 313 | PVFS_object_attr seg_attr; |
|---|
| 314 | PVFS_object_ref seg_starting_refn; |
|---|
| 315 | PVFS_object_ref seg_resolved_refn; |
|---|
| 316 | } PINT_client_lookup_sm_segment; |
|---|
| 317 | |
|---|
| 318 | typedef struct |
|---|
| 319 | { |
|---|
| 320 | int total_segments; |
|---|
| 321 | int current_segment; |
|---|
| 322 | PINT_client_lookup_sm_segment segments[MAX_LOOKUP_SEGMENTS]; |
|---|
| 323 | PVFS_object_ref ctx_starting_refn; |
|---|
| 324 | PVFS_object_ref ctx_resolved_refn; |
|---|
| 325 | } PINT_client_lookup_sm_ctx; |
|---|
| 326 | |
|---|
| 327 | struct PINT_client_lookup_sm |
|---|
| 328 | { |
|---|
| 329 | char *orig_pathname; /* input parameter */ |
|---|
| 330 | PVFS_object_ref starting_refn; /* input parameter */ |
|---|
| 331 | PVFS_sysresp_lookup *lookup_resp; /* in/out parameter*/ |
|---|
| 332 | int follow_link; /* input parameter */ |
|---|
| 333 | int skipped_final_resolution; |
|---|
| 334 | int current_context; |
|---|
| 335 | int context_count; |
|---|
| 336 | PINT_client_lookup_sm_ctx * contexts; |
|---|
| 337 | }; |
|---|
| 338 | |
|---|
| 339 | struct PINT_client_rename_sm |
|---|
| 340 | { |
|---|
| 341 | char *entries[2]; /* old/new input entry names */ |
|---|
| 342 | PVFS_object_ref parent_refns[2]; /* old/new input parent refns */ |
|---|
| 343 | |
|---|
| 344 | PVFS_object_ref refns[2]; /* old/new object refns */ |
|---|
| 345 | PVFS_ds_type types[2]; /* old/new object types */ |
|---|
| 346 | PVFS_handle dirent_handle[2]; /* old/new dirent handles for parent dirs */ |
|---|
| 347 | int retry_count; |
|---|
| 348 | int stored_error_code; |
|---|
| 349 | int rmdirent_index; |
|---|
| 350 | int target_dirent_exists; |
|---|
| 351 | PVFS_handle old_dirent_handle; |
|---|
| 352 | }; |
|---|
| 353 | |
|---|
| 354 | struct PINT_client_mgmt_setparam_list_sm |
|---|
| 355 | { |
|---|
| 356 | PVFS_fs_id fs_id; |
|---|
| 357 | enum PVFS_server_param param; |
|---|
| 358 | struct PVFS_mgmt_setparam_value *value; |
|---|
| 359 | PVFS_id_gen_t *addr_array; |
|---|
| 360 | int count; |
|---|
| 361 | int *root_check_status_array; |
|---|
| 362 | PVFS_error_details *details; |
|---|
| 363 | }; |
|---|
| 364 | |
|---|
| 365 | struct PINT_client_mgmt_statfs_list_sm |
|---|
| 366 | { |
|---|
| 367 | PVFS_fs_id fs_id; |
|---|
| 368 | struct PVFS_mgmt_server_stat *stat_array; |
|---|
| 369 | int count; |
|---|
| 370 | PVFS_id_gen_t *addr_array; |
|---|
| 371 | PVFS_error_details *details; |
|---|
| 372 | PVFS_sysresp_statfs* resp; /* ignored by mgmt functions */ |
|---|
| 373 | }; |
|---|
| 374 | |
|---|
| 375 | struct PINT_client_mgmt_perf_mon_list_sm |
|---|
| 376 | { |
|---|
| 377 | PVFS_fs_id fs_id; |
|---|
| 378 | struct PVFS_mgmt_perf_stat **perf_matrix; |
|---|
| 379 | uint64_t *end_time_ms_array; |
|---|
| 380 | int server_count; |
|---|
| 381 | int history_count; |
|---|
| 382 | PVFS_id_gen_t *addr_array; |
|---|
| 383 | uint32_t *next_id_array; |
|---|
| 384 | PVFS_error_details *details; |
|---|
| 385 | }; |
|---|
| 386 | |
|---|
| 387 | struct PINT_client_mgmt_event_mon_list_sm |
|---|
| 388 | { |
|---|
| 389 | PVFS_fs_id fs_id; |
|---|
| 390 | struct PVFS_mgmt_event **event_matrix; |
|---|
| 391 | int server_count; |
|---|
| 392 | int event_count; |
|---|
| 393 | PVFS_id_gen_t *addr_array; |
|---|
| 394 | PVFS_error_details *details; |
|---|
| 395 | }; |
|---|
| 396 | |
|---|
| 397 | struct PINT_client_mgmt_iterate_handles_list_sm |
|---|
| 398 | { |
|---|
| 399 | PVFS_fs_id fs_id; |
|---|
| 400 | int server_count; |
|---|
| 401 | PVFS_id_gen_t *addr_array; |
|---|
| 402 | PVFS_handle **handle_matrix; |
|---|
| 403 | int *handle_count_array; |
|---|
| 404 | PVFS_ds_position *position_array; |
|---|
| 405 | PVFS_error_details *details; |
|---|
| 406 | int flags; |
|---|
| 407 | }; |
|---|
| 408 | |
|---|
| 409 | struct PINT_client_mgmt_get_dfile_array_sm |
|---|
| 410 | { |
|---|
| 411 | PVFS_handle *dfile_array; |
|---|
| 412 | int dfile_count; |
|---|
| 413 | }; |
|---|
| 414 | |
|---|
| 415 | struct PINT_client_mgmt_get_dirdata_array_sm |
|---|
| 416 | { |
|---|
| 417 | PVFS_handle *dirdata_array; |
|---|
| 418 | int dirdata_count; |
|---|
| 419 | }; |
|---|
| 420 | |
|---|
| 421 | struct PINT_client_truncate_sm |
|---|
| 422 | { |
|---|
| 423 | PVFS_size size; /* new logical size of object*/ |
|---|
| 424 | }; |
|---|
| 425 | |
|---|
| 426 | struct PINT_server_get_config_sm |
|---|
| 427 | { |
|---|
| 428 | struct server_configuration_s *config; |
|---|
| 429 | struct PVFS_sys_mntent *mntent; |
|---|
| 430 | char *fs_config_buf; |
|---|
| 431 | uint32_t fs_config_buf_size; |
|---|
| 432 | int persist_config_buffers; |
|---|
| 433 | int free_config_flag; |
|---|
| 434 | }; |
|---|
| 435 | |
|---|
| 436 | struct PINT_server_fetch_config_sm_state |
|---|
| 437 | { |
|---|
| 438 | int nservers; |
|---|
| 439 | PVFS_BMI_addr_t *addr_array; |
|---|
| 440 | char **fs_config_bufs; |
|---|
| 441 | int *fs_config_buf_size; |
|---|
| 442 | int result_count; /* number of servers that actually responded */ |
|---|
| 443 | int* result_indexes; /* index into fs_config_bufs of valid responses */ |
|---|
| 444 | }; |
|---|
| 445 | |
|---|
| 446 | |
|---|
| 447 | |
|---|
| 448 | /* flag to disable cached lookup during getattr nested sm */ |
|---|
| 449 | #define PINT_SM_GETATTR_BYPASS_CACHE 1 |
|---|
| 450 | |
|---|
| 451 | typedef struct PINT_sm_getattr_state |
|---|
| 452 | { |
|---|
| 453 | PVFS_object_ref object_ref; |
|---|
| 454 | |
|---|
| 455 | /* request sys attrmask. Some combination of |
|---|
| 456 | * PVFS_ATTR_SYS_* |
|---|
| 457 | */ |
|---|
| 458 | uint32_t req_attrmask; |
|---|
| 459 | |
|---|
| 460 | /* |
|---|
| 461 | Either from the acache or full getattr op, this is the resuling |
|---|
| 462 | attribute that can be used by calling state machines |
|---|
| 463 | */ |
|---|
| 464 | PVFS_object_attr attr; |
|---|
| 465 | |
|---|
| 466 | |
|---|
| 467 | /* mirror retry information */ |
|---|
| 468 | PINT_client_getattr_mirror_ctx *mir_ctx_array; |
|---|
| 469 | uint32_t mir_ctx_count; |
|---|
| 470 | uint32_t retry_count; |
|---|
| 471 | uint32_t *index_to_server; |
|---|
| 472 | |
|---|
| 473 | PVFS_ds_type ref_type; |
|---|
| 474 | |
|---|
| 475 | /* used with sys-readdir to get dirent_count of all dirdata handles, |
|---|
| 476 | * will be set to 0 in PINT_SM_GETATTR_STATE_FILL, |
|---|
| 477 | * now only used with sys-readdir.sm */ |
|---|
| 478 | int keep_size_array; |
|---|
| 479 | int *active_dirdata_index; |
|---|
| 480 | |
|---|
| 481 | PVFS_size * size_array; |
|---|
| 482 | PVFS_size size; |
|---|
| 483 | |
|---|
| 484 | int flags; |
|---|
| 485 | |
|---|
| 486 | } PINT_sm_getattr_state; |
|---|
| 487 | |
|---|
| 488 | #define PINT_SM_GETATTR_STATE_FILL(_state, _objref, _mask, _reftype, _flags) \ |
|---|
| 489 | do { \ |
|---|
| 490 | memset(&(_state), 0, sizeof(PINT_sm_getattr_state)); \ |
|---|
| 491 | (_state).object_ref.fs_id = (_objref).fs_id; \ |
|---|
| 492 | (_state).object_ref.handle = (_objref).handle; \ |
|---|
| 493 | (_state).req_attrmask = _mask; \ |
|---|
| 494 | (_state).ref_type = _reftype; \ |
|---|
| 495 | (_state).flags = _flags; \ |
|---|
| 496 | (_state).keep_size_array = 0; \ |
|---|
| 497 | } while(0) |
|---|
| 498 | |
|---|
| 499 | #define PINT_SM_GETATTR_STATE_CLEAR(_state) \ |
|---|
| 500 | do { \ |
|---|
| 501 | PINT_free_object_attr(&(_state).attr); \ |
|---|
| 502 | memset(&(_state), 0, sizeof(PINT_sm_getattr_state)); \ |
|---|
| 503 | } while(0) |
|---|
| 504 | |
|---|
| 505 | #define PINT_SM_DATAFILE_SIZE_ARRAY_INIT(_array, _count) \ |
|---|
| 506 | do { \ |
|---|
| 507 | (*(_array)) = malloc(sizeof(PVFS_size) * (_count)); \ |
|---|
| 508 | memset(*(_array), 0, (sizeof(PVFS_size) * (_count))); \ |
|---|
| 509 | } while(0) |
|---|
| 510 | |
|---|
| 511 | #define PINT_SM_DATAFILE_SIZE_ARRAY_DESTROY(_array) \ |
|---|
| 512 | do { \ |
|---|
| 513 | free(*(_array)); \ |
|---|
| 514 | *(_array) = NULL; \ |
|---|
| 515 | } while(0) |
|---|
| 516 | |
|---|
| 517 | struct PINT_client_geteattr_sm |
|---|
| 518 | { |
|---|
| 519 | int32_t nkey; |
|---|
| 520 | PVFS_ds_keyval *key_array; |
|---|
| 521 | PVFS_size *size_array; |
|---|
| 522 | PVFS_sysresp_geteattr *resp_p; |
|---|
| 523 | }; |
|---|
| 524 | |
|---|
| 525 | struct PINT_client_seteattr_sm |
|---|
| 526 | { |
|---|
| 527 | int32_t nkey; |
|---|
| 528 | int32_t flags; /* flags specify if attrs should not exist (XATTR_CREATE) or |
|---|
| 529 | if they should exist (XATTR_REPLACE) or neither */ |
|---|
| 530 | PVFS_ds_keyval *key_array; |
|---|
| 531 | PVFS_ds_keyval *val_array; |
|---|
| 532 | }; |
|---|
| 533 | |
|---|
| 534 | struct PINT_client_deleattr_sm |
|---|
| 535 | { |
|---|
| 536 | PVFS_ds_keyval *key_p; |
|---|
| 537 | }; |
|---|
| 538 | |
|---|
| 539 | struct PINT_client_listeattr_sm |
|---|
| 540 | { |
|---|
| 541 | PVFS_ds_position pos_token; /* input parameter */ |
|---|
| 542 | int32_t nkey; /* input parameter */ |
|---|
| 543 | PVFS_size *size_array; /* Input/Output */ |
|---|
| 544 | PVFS_sysresp_listeattr *resp_p; /* Output */ |
|---|
| 545 | }; |
|---|
| 546 | |
|---|
| 547 | struct PINT_client_perf_count_timer_sm |
|---|
| 548 | { |
|---|
| 549 | unsigned int *interval_secs; |
|---|
| 550 | struct PINT_perf_counter *pc; |
|---|
| 551 | }; |
|---|
| 552 | |
|---|
| 553 | struct PINT_client_job_timer_sm |
|---|
| 554 | { |
|---|
| 555 | job_id_t job_id; |
|---|
| 556 | }; |
|---|
| 557 | |
|---|
| 558 | struct PINT_sysdev_unexp_sm |
|---|
| 559 | { |
|---|
| 560 | struct PINT_dev_unexp_info *info; |
|---|
| 561 | }; |
|---|
| 562 | |
|---|
| 563 | typedef struct |
|---|
| 564 | { |
|---|
| 565 | PVFS_dirent **dirent_array; |
|---|
| 566 | uint32_t *dirent_outcount; |
|---|
| 567 | PVFS_ds_position *token; |
|---|
| 568 | uint64_t *directory_version; |
|---|
| 569 | PVFS_ds_position pos_token; /* input/output parameter */ |
|---|
| 570 | int32_t dirent_limit; /* input parameter */ |
|---|
| 571 | int32_t dirdata_index; /* input parameter */ |
|---|
| 572 | } PINT_sm_readdir_state; |
|---|
| 573 | |
|---|
| 574 | typedef struct PINT_client_sm |
|---|
| 575 | { |
|---|
| 576 | /* this code removed and corresponding fields added to the generic |
|---|
| 577 | * state machine code in the PINT_smcb struct |
|---|
| 578 | */ |
|---|
| 579 | /* used internally by client-state-machine.c */ |
|---|
| 580 | PVFS_sys_op_id sys_op_id; |
|---|
| 581 | void *user_ptr; |
|---|
| 582 | |
|---|
| 583 | PINT_event_id event_id; |
|---|
| 584 | |
|---|
| 585 | /* stores the final operation error code on operation exit */ |
|---|
| 586 | PVFS_error error_code; |
|---|
| 587 | |
|---|
| 588 | int comp_ct; /* used to keep up with completion of multiple |
|---|
| 589 | * jobs for some states; typically set and |
|---|
| 590 | * then decremented to zero as jobs complete */ |
|---|
| 591 | |
|---|
| 592 | /* generic getattr used with getattr sub state machines */ |
|---|
| 593 | PINT_sm_getattr_state getattr; |
|---|
| 594 | /* generic dirent array used by both readdir and readdirplus state machines */ |
|---|
| 595 | PINT_sm_readdir_state readdir; |
|---|
| 596 | |
|---|
| 597 | /* fetch_config state used by the nested fetch config state machines */ |
|---|
| 598 | struct PINT_server_fetch_config_sm_state fetch_config; |
|---|
| 599 | |
|---|
| 600 | PVFS_hint hints; |
|---|
| 601 | |
|---|
| 602 | PINT_sm_msgarray_op msgarray_op; |
|---|
| 603 | |
|---|
| 604 | PVFS_object_ref object_ref; |
|---|
| 605 | PVFS_object_ref parent_ref; |
|---|
| 606 | |
|---|
| 607 | |
|---|
| 608 | PVFS_credentials *cred_p; |
|---|
| 609 | union |
|---|
| 610 | { |
|---|
| 611 | struct PINT_client_remove_sm remove; |
|---|
| 612 | struct PINT_client_create_sm create; |
|---|
| 613 | struct PINT_client_mkdir_sm mkdir; |
|---|
| 614 | struct PINT_client_symlink_sm sym; |
|---|
| 615 | struct PINT_client_getattr_sm getattr; |
|---|
| 616 | struct PINT_client_setattr_sm setattr; |
|---|
| 617 | struct PINT_client_io_sm io; |
|---|
| 618 | struct PINT_client_flush_sm flush; |
|---|
| 619 | struct PINT_client_readdir_sm readdir; |
|---|
| 620 | struct PINT_client_readdirplus_sm readdirplus; |
|---|
| 621 | struct PINT_client_lookup_sm lookup; |
|---|
| 622 | struct PINT_client_rename_sm rename; |
|---|
| 623 | struct PINT_client_mgmt_setparam_list_sm setparam_list; |
|---|
| 624 | struct PINT_client_truncate_sm truncate; |
|---|
| 625 | struct PINT_client_mgmt_statfs_list_sm statfs_list; |
|---|
| 626 | struct PINT_client_mgmt_perf_mon_list_sm perf_mon_list; |
|---|
| 627 | struct PINT_client_mgmt_event_mon_list_sm event_mon_list; |
|---|
| 628 | struct PINT_client_mgmt_iterate_handles_list_sm iterate_handles_list; |
|---|
| 629 | struct PINT_client_mgmt_get_dfile_array_sm get_dfile_array; |
|---|
| 630 | struct PINT_client_mgmt_get_dirdata_array_sm get_dirdata_array; |
|---|
| 631 | struct PINT_client_mgmt_remove_dirent_sm mgmt_remove_dirent; |
|---|
| 632 | struct PINT_client_mgmt_create_dirent_sm mgmt_create_dirent; |
|---|
| 633 | struct PINT_client_mgmt_get_dirdata_handle_sm mgmt_get_dirdata_handle; |
|---|
| 634 | struct PINT_server_get_config_sm get_config; |
|---|
| 635 | struct PINT_client_geteattr_sm geteattr; |
|---|
| 636 | struct PINT_client_seteattr_sm seteattr; |
|---|
| 637 | struct PINT_client_deleattr_sm deleattr; |
|---|
| 638 | struct PINT_client_listeattr_sm listeattr; |
|---|
| 639 | struct PINT_client_perf_count_timer_sm perf_count_timer; |
|---|
| 640 | struct PINT_sysdev_unexp_sm sysdev_unexp; |
|---|
| 641 | struct PINT_client_job_timer_sm job_timer; |
|---|
| 642 | } u; |
|---|
| 643 | } PINT_client_sm; |
|---|
| 644 | |
|---|
| 645 | /* sysint post/test functions */ |
|---|
| 646 | PVFS_error PINT_client_state_machine_post( |
|---|
| 647 | PINT_smcb *smcb, |
|---|
| 648 | PVFS_sys_op_id *op_id, |
|---|
| 649 | void *user_ptr); |
|---|
| 650 | |
|---|
| 651 | PVFS_error PINT_client_state_machine_release( |
|---|
| 652 | PINT_smcb * smcb); |
|---|
| 653 | |
|---|
| 654 | PVFS_error PINT_sys_dev_unexp( |
|---|
| 655 | struct PINT_dev_unexp_info *info, |
|---|
| 656 | job_status_s *jstat, |
|---|
| 657 | PVFS_sys_op_id *op_id, |
|---|
| 658 | void *user_ptr); |
|---|
| 659 | |
|---|
| 660 | PVFS_error PINT_client_state_machine_test( |
|---|
| 661 | PVFS_sys_op_id op_id, |
|---|
| 662 | int *error_code); |
|---|
| 663 | |
|---|
| 664 | PVFS_error PINT_client_state_machine_testsome( |
|---|
| 665 | PVFS_sys_op_id *op_id_array, |
|---|
| 666 | int *op_count, /* in/out */ |
|---|
| 667 | void **user_ptr_array, |
|---|
| 668 | int *error_code_array, |
|---|
| 669 | int timeout_ms); |
|---|
| 670 | |
|---|
| 671 | /* exposed wrappers around the id-generator code */ |
|---|
| 672 | static inline int PINT_id_gen_safe_register( |
|---|
| 673 | PVFS_sys_op_id *new_id, |
|---|
| 674 | void *item) |
|---|
| 675 | { |
|---|
| 676 | return id_gen_safe_register(new_id, item); |
|---|
| 677 | } |
|---|
| 678 | |
|---|
| 679 | static inline void *PINT_id_gen_safe_lookup( |
|---|
| 680 | PVFS_sys_op_id id) |
|---|
| 681 | { |
|---|
| 682 | return id_gen_safe_lookup(id); |
|---|
| 683 | } |
|---|
| 684 | |
|---|
| 685 | static inline int PINT_id_gen_safe_unregister( |
|---|
| 686 | PVFS_sys_op_id id) |
|---|
| 687 | { |
|---|
| 688 | return id_gen_safe_unregister(id); |
|---|
| 689 | } |
|---|
| 690 | |
|---|
| 691 | /* debugging method for getting a string macthing the op_type */ |
|---|
| 692 | const char *PINT_client_get_name_str(int op_type); |
|---|
| 693 | |
|---|
| 694 | /* used with post call to tell the system what state machine to use |
|---|
| 695 | * when processing a new PINT_client_sm structure. |
|---|
| 696 | */ |
|---|
| 697 | enum |
|---|
| 698 | { |
|---|
| 699 | PVFS_SYS_REMOVE = 1, |
|---|
| 700 | PVFS_SYS_CREATE = 2, |
|---|
| 701 | PVFS_SYS_MKDIR = 3, |
|---|
| 702 | PVFS_SYS_SYMLINK = 4, |
|---|
| 703 | PVFS_SYS_GETATTR = 5, |
|---|
| 704 | PVFS_SYS_IO = 6, |
|---|
| 705 | PVFS_SYS_FLUSH = 7, |
|---|
| 706 | PVFS_SYS_TRUNCATE = 8, |
|---|
| 707 | PVFS_SYS_READDIR = 9, |
|---|
| 708 | PVFS_SYS_SETATTR = 10, |
|---|
| 709 | PVFS_SYS_LOOKUP = 11, |
|---|
| 710 | PVFS_SYS_RENAME = 12, |
|---|
| 711 | PVFS_SYS_GETEATTR = 13, |
|---|
| 712 | PVFS_SYS_SETEATTR = 14, |
|---|
| 713 | PVFS_SYS_DELEATTR = 15, |
|---|
| 714 | PVFS_SYS_LISTEATTR = 16, |
|---|
| 715 | PVFS_SYS_SMALL_IO = 17, |
|---|
| 716 | PVFS_SYS_STATFS = 18, |
|---|
| 717 | PVFS_SYS_FS_ADD = 19, |
|---|
| 718 | PVFS_SYS_READDIRPLUS = 20, |
|---|
| 719 | PVFS_MGMT_SETPARAM_LIST = 70, |
|---|
| 720 | PVFS_MGMT_NOOP = 71, |
|---|
| 721 | PVFS_MGMT_STATFS_LIST = 72, |
|---|
| 722 | PVFS_MGMT_PERF_MON_LIST = 73, |
|---|
| 723 | PVFS_MGMT_ITERATE_HANDLES_LIST = 74, |
|---|
| 724 | PVFS_MGMT_GET_DFILE_ARRAY = 75, |
|---|
| 725 | PVFS_MGMT_EVENT_MON_LIST = 76, |
|---|
| 726 | PVFS_MGMT_REMOVE_OBJECT = 77, |
|---|
| 727 | PVFS_MGMT_REMOVE_DIRENT = 78, |
|---|
| 728 | PVFS_MGMT_CREATE_DIRENT = 79, |
|---|
| 729 | PVFS_MGMT_GET_DIRDATA_HANDLE = 80, |
|---|
| 730 | PVFS_MGMT_GET_DIRDATA_ARRAY = 81, |
|---|
| 731 | PVFS_SERVER_GET_CONFIG = 200, |
|---|
| 732 | PVFS_CLIENT_JOB_TIMER = 300, |
|---|
| 733 | PVFS_CLIENT_PERF_COUNT_TIMER = 301, |
|---|
| 734 | PVFS_DEV_UNEXPECTED = 400 |
|---|
| 735 | }; |
|---|
| 736 | |
|---|
| 737 | #define PVFS_OP_SYS_MAXVALID 21 |
|---|
| 738 | #define PVFS_OP_SYS_MAXVAL 69 |
|---|
| 739 | #define PVFS_OP_MGMT_MAXVALID 81 |
|---|
| 740 | #define PVFS_OP_MGMT_MAXVAL 199 |
|---|
| 741 | |
|---|
| 742 | int PINT_client_io_cancel(job_id_t id); |
|---|
| 743 | |
|---|
| 744 | /* internal non-blocking helper methods */ |
|---|
| 745 | int PINT_client_wait_internal( |
|---|
| 746 | PVFS_sys_op_id op_id, |
|---|
| 747 | const char *in_op_str, |
|---|
| 748 | int *out_error, |
|---|
| 749 | const char *in_class_str); |
|---|
| 750 | |
|---|
| 751 | void PINT_sys_release(PVFS_sys_op_id op_id); |
|---|
| 752 | |
|---|
| 753 | void PINT_mgmt_release(PVFS_mgmt_op_id op_id); |
|---|
| 754 | |
|---|
| 755 | /* internal helper macros */ |
|---|
| 756 | #define PINT_init_sysint_credentials(sm_p_cred_p, user_cred_p)\ |
|---|
| 757 | do { \ |
|---|
| 758 | if (user_cred_p == NULL) \ |
|---|
| 759 | { \ |
|---|
| 760 | gossip_lerr("Invalid user credentials! (nil)\n"); \ |
|---|
| 761 | free(sm_p); \ |
|---|
| 762 | return -PVFS_EINVAL; \ |
|---|
| 763 | } \ |
|---|
| 764 | sm_p_cred_p = PVFS_util_dup_credentials(user_cred_p); \ |
|---|
| 765 | if (!sm_p_cred_p) \ |
|---|
| 766 | { \ |
|---|
| 767 | gossip_lerr("Failed to copy user credentials\n"); \ |
|---|
| 768 | free(sm_p); \ |
|---|
| 769 | return -PVFS_ENOMEM; \ |
|---|
| 770 | } \ |
|---|
| 771 | } while(0) |
|---|
| 772 | |
|---|
| 773 | #define PINT_init_msgarray_params(client_sm_p, __fsid) \ |
|---|
| 774 | do { \ |
|---|
| 775 | PINT_sm_msgpair_params *mpp = &client_sm_p->msgarray_op.params; \ |
|---|
| 776 | struct server_configuration_s *server_config = \ |
|---|
| 777 | PINT_get_server_config_struct(__fsid); \ |
|---|
| 778 | mpp->job_context = pint_client_sm_context; \ |
|---|
| 779 | if (server_config) \ |
|---|
| 780 | { \ |
|---|
| 781 | mpp->job_timeout = server_config->client_job_bmi_timeout; \ |
|---|
| 782 | mpp->retry_limit = server_config->client_retry_limit; \ |
|---|
| 783 | mpp->retry_delay = server_config->client_retry_delay_ms; \ |
|---|
| 784 | } \ |
|---|
| 785 | else \ |
|---|
| 786 | { \ |
|---|
| 787 | mpp->job_timeout = PVFS2_CLIENT_JOB_BMI_TIMEOUT_DEFAULT; \ |
|---|
| 788 | mpp->retry_limit = PVFS2_CLIENT_RETRY_LIMIT_DEFAULT; \ |
|---|
| 789 | mpp->retry_delay = PVFS2_CLIENT_RETRY_DELAY_MS_DEFAULT; \ |
|---|
| 790 | } \ |
|---|
| 791 | PINT_put_server_config_struct(server_config); \ |
|---|
| 792 | } while(0) |
|---|
| 793 | |
|---|
| 794 | struct PINT_client_op_entry_s |
|---|
| 795 | { |
|---|
| 796 | struct PINT_state_machine_s * sm; |
|---|
| 797 | }; |
|---|
| 798 | |
|---|
| 799 | extern struct PINT_client_op_entry_s PINT_client_sm_sys_table[]; |
|---|
| 800 | extern struct PINT_client_op_entry_s PINT_client_sm_mgmt_table[]; |
|---|
| 801 | |
|---|
| 802 | /* system interface function state machines */ |
|---|
| 803 | extern struct PINT_state_machine_s pvfs2_client_remove_sm; |
|---|
| 804 | extern struct PINT_state_machine_s pvfs2_client_create_sm; |
|---|
| 805 | extern struct PINT_state_machine_s pvfs2_client_mkdir_sm; |
|---|
| 806 | extern struct PINT_state_machine_s pvfs2_client_symlink_sm; |
|---|
| 807 | extern struct PINT_state_machine_s pvfs2_client_sysint_getattr_sm; |
|---|
| 808 | extern struct PINT_state_machine_s pvfs2_client_getattr_sm; |
|---|
| 809 | extern struct PINT_state_machine_s pvfs2_client_datafile_getattr_sizes_sm; |
|---|
| 810 | extern struct PINT_state_machine_s pvfs2_client_setattr_sm; |
|---|
| 811 | extern struct PINT_state_machine_s pvfs2_client_io_sm; |
|---|
| 812 | extern struct PINT_state_machine_s pvfs2_client_small_io_sm; |
|---|
| 813 | extern struct PINT_state_machine_s pvfs2_client_flush_sm; |
|---|
| 814 | extern struct PINT_state_machine_s pvfs2_client_sysint_readdir_sm; |
|---|
| 815 | extern struct PINT_state_machine_s pvfs2_client_readdir_sm; |
|---|
| 816 | extern struct PINT_state_machine_s pvfs2_client_readdirplus_sm; |
|---|
| 817 | extern struct PINT_state_machine_s pvfs2_client_lookup_sm; |
|---|
| 818 | extern struct PINT_state_machine_s pvfs2_client_rename_sm; |
|---|
| 819 | extern struct PINT_state_machine_s pvfs2_client_truncate_sm; |
|---|
| 820 | extern struct PINT_state_machine_s pvfs2_sysdev_unexp_sm; |
|---|
| 821 | extern struct PINT_state_machine_s pvfs2_client_job_timer_sm; |
|---|
| 822 | extern struct PINT_state_machine_s pvfs2_client_perf_count_timer_sm; |
|---|
| 823 | extern struct PINT_state_machine_s pvfs2_server_get_config_sm; |
|---|
| 824 | extern struct PINT_state_machine_s pvfs2_server_fetch_config_sm; |
|---|
| 825 | extern struct PINT_state_machine_s pvfs2_client_mgmt_setparam_list_sm; |
|---|
| 826 | extern struct PINT_state_machine_s pvfs2_client_mgmt_statfs_list_sm; |
|---|
| 827 | extern struct PINT_state_machine_s pvfs2_client_mgmt_perf_mon_list_sm; |
|---|
| 828 | extern struct PINT_state_machine_s pvfs2_client_mgmt_event_mon_list_sm; |
|---|
| 829 | extern struct PINT_state_machine_s pvfs2_client_mgmt_iterate_handles_list_sm; |
|---|
| 830 | extern struct PINT_state_machine_s pvfs2_client_mgmt_get_dfile_array_sm; |
|---|
| 831 | extern struct PINT_state_machine_s pvfs2_client_mgmt_get_dirdata_array_sm; |
|---|
| 832 | extern struct PINT_state_machine_s pvfs2_client_mgmt_noop_sm; |
|---|
| 833 | extern struct PINT_state_machine_s pvfs2_client_mgmt_remove_object_sm; |
|---|
| 834 | extern struct PINT_state_machine_s pvfs2_client_mgmt_remove_dirent_sm; |
|---|
| 835 | extern struct PINT_state_machine_s pvfs2_client_mgmt_create_dirent_sm; |
|---|
| 836 | extern struct PINT_state_machine_s pvfs2_client_mgmt_get_dirdata_handle_sm; |
|---|
| 837 | extern struct PINT_state_machine_s pvfs2_client_get_eattr_sm; |
|---|
| 838 | extern struct PINT_state_machine_s pvfs2_client_set_eattr_sm; |
|---|
| 839 | extern struct PINT_state_machine_s pvfs2_client_del_eattr_sm; |
|---|
| 840 | extern struct PINT_state_machine_s pvfs2_client_list_eattr_sm; |
|---|
| 841 | extern struct PINT_state_machine_s pvfs2_client_statfs_sm; |
|---|
| 842 | extern struct PINT_state_machine_s pvfs2_fs_add_sm; |
|---|
| 843 | |
|---|
| 844 | /* nested state machines (helpers) */ |
|---|
| 845 | extern struct PINT_state_machine_s pvfs2_client_lookup_ncache_sm; |
|---|
| 846 | extern struct PINT_state_machine_s pvfs2_client_remove_helper_sm; |
|---|
| 847 | extern struct PINT_state_machine_s pvfs2_client_mgmt_statfs_list_nested_sm; |
|---|
| 848 | extern struct PINT_state_machine_s pvfs2_server_get_config_nested_sm; |
|---|
| 849 | extern struct PINT_state_machine_s pvfs2_server_fetch_config_nested_sm; |
|---|
| 850 | |
|---|
| 851 | #include "state-machine.h" |
|---|
| 852 | |
|---|
| 853 | /* method for lookup up SM from OP */ |
|---|
| 854 | struct PINT_state_machine_s *client_op_state_get_machine(int); |
|---|
| 855 | int client_state_machine_terminate(struct PINT_smcb *, job_status_s *); |
|---|
| 856 | |
|---|
| 857 | #endif /* __PVFS2_CLIENT_STATE_MACHINE_H */ |
|---|
| 858 | |
|---|
| 859 | /* |
|---|
| 860 | * Local variables: |
|---|
| 861 | * c-indent-level: 4 |
|---|
| 862 | * c-basic-offset: 4 |
|---|
| 863 | * End: |
|---|
| 864 | * |
|---|
| 865 | * vim: ts=8 sts=4 sw=4 expandtab |
|---|
| 866 | */ |
|---|