Context Navigation

pvfs2-server.c @ 8863

Revision 8863, 90.0 KB (checked in by mtmoore, 2 years ago)
don't try to shutdown the server if the help argument is passed to pvfs2-server

Line
1	/*
2	* (C) 2001 Clemson University and The University of Chicago
3	*
4	* See COPYING in top-level directory.
5	*/
6
7	#include <errno.h>
8	#include <string.h>
9	#include <stdlib.h>
10	#include <stdio.h>
11	#include <stdarg.h>
12	#include <unistd.h>
13	#include <fcntl.h>
14	#include <sys/types.h>
15	#include <sys/stat.h>
16	#include <signal.h>
17	#include <assert.h>
18	#include <getopt.h>
19
20	#ifdef __PVFS2_SEGV_BACKTRACE__
21	#include <execinfo.h>
22	#include <ucontext.h>
23	#endif
24
25	#define __PINT_REQPROTO_ENCODE_FUNCS_C
26
27	#include "bmi.h"
28	#include "gossip.h"
29	#include "job.h"
30	#include "trove.h"
31	#include "pvfs2-debug.h"
32	#include "pvfs2-storage.h"
33	#include "PINT-reqproto-encode.h"
34	#include "pvfs2-server.h"
35	#include "state-machine.h"
36	#include "mkspace.h"
37	#include "server-config.h"
38	#include "quicklist.h"
39	#include "pint-dist-utils.h"
40	#include "pint-perf-counter.h"
41	#include "id-generator.h"
42	#include "job-time-mgr.h"
43	#include "pint-cached-config.h"
44	#include "pvfs2-internal.h"
45	#include "src/server/request-scheduler/request-scheduler.h"
46	#include "pint-event.h"
47	#include "pint-util.h"
48
49	#ifndef PVFS2_VERSION
50	#define PVFS2_VERSION "Unknown"
51	#endif
52
53	#ifdef __PVFS2_TROVE_THREADED__
54	#ifdef __PVFS2_TROVE_AIO_THREADED__
55	#define SERVER_STORAGE_MODE "aio-threaded"
56	#else
57	#define SERVER_STORAGE_MODE "threaded"
58	#endif
59	#else
60	#define SERVER_STORAGE_MODE "non-threaded"
61	#endif
62
63	#define PVFS2_VERSION_REQUEST 0xFF
64	#define PVFS2_HELP 0xFE
65
66	/* this controls how many jobs we will test for per job_testcontext()
67	* call. NOTE: this is currently independent of the config file
68	* parameter that governs how many unexpected BMI jobs are kept posted
69	* at any given time
70	*/
71	#define PVFS_SERVER_TEST_COUNT 64
72
73	/* track performance counters for the server */
74	static struct PINT_perf_key server_keys[] =
75	{
76	{"bytes read", PINT_PERF_READ, 0},
77	{"bytes written", PINT_PERF_WRITE, 0},
78	{"metadata reads", PINT_PERF_METADATA_READ, PINT_PERF_PRESERVE},
79	{"metadata writes", PINT_PERF_METADATA_WRITE, PINT_PERF_PRESERVE},
80	{"metadata dspace ops", PINT_PERF_METADATA_DSPACE_OPS, PINT_PERF_PRESERVE},
81	{"metadata keyval ops", PINT_PERF_METADATA_KEYVAL_OPS, PINT_PERF_PRESERVE},
82	{"request scheduler", PINT_PERF_REQSCHED, PINT_PERF_PRESERVE},
83	{NULL, 0, 0},
84	};
85
86	/* For the switch statement to know what interfaces to shutdown */
87	static PINT_server_status_flag server_status_flag;
88
89	/* All parameters read in from the configuration file */
90	static struct server_configuration_s server_config;
91
92	/* A flag to stop the main loop from processing and handle the signal
93	* after all threads complete and are no longer blocking.
94	*/
95	static int signal_recvd_flag = 0;
96	static pid_t server_controlling_pid = 0;
97
98	static PINT_event_id PINT_sm_event_id;
99
100	/* A list of all serv_op's posted for unexpected message alone */
101	QLIST_HEAD(posted_sop_list);
102	/* A list of all serv_op's posted for expected messages alone */
103	QLIST_HEAD(inprogress_sop_list);
104	/* A list of all serv_op's that are started automatically without requests */
105	static QLIST_HEAD(noreq_sop_list);
106
107	/* this is used externally by some server state machines */
108	job_context_id server_job_context = -1;
109
110	typedef struct
111	{
112	int server_remove_storage_space;
113	int server_create_storage_space;
114	int server_background;
115	char *pidfile;
116	char *server_alias;
117	} options_t;
118
119	static options_t s_server_options = { 0, 0, 1, NULL, NULL};
120	static char fs_conf[PATH_MAX];
121	static char startup_cwd[PATH_MAX];
122
123	/* each of the elements in this array consists of a string and its length.
124	* we're able to use sizeof here because sizeof an inlined string ("") gives
125	* the length of the string with the null terminator
126	*/
127	PINT_server_trove_keys_s Trove_Common_Keys[] =
128	{
129	{ROOT_HANDLE_KEYSTR, ROOT_HANDLE_KEYLEN},
130	{DIRECTORY_ENTRY_KEYSTR, DIRECTORY_ENTRY_KEYLEN},
131	{DATAFILE_HANDLES_KEYSTR, DATAFILE_HANDLES_KEYLEN},
132	{METAFILE_DIST_KEYSTR, METAFILE_DIST_KEYLEN},
133	{SYMLINK_TARGET_KEYSTR, SYMLINK_TARGET_KEYLEN},
134	{METAFILE_LAYOUT_KEYSTR, METAFILE_LAYOUT_KEYLEN},
135	{NUM_DFILES_REQ_KEYSTR, NUM_DFILES_REQ_KEYLEN}
136	};
137
138	/* These three are used continuously in our wait loop. They could be
139	* relatively large, so rather than allocate them on the stack, we'll
140	* make them dynamically allocated globals.
141	*/
142	static job_id_t *server_job_id_array = NULL;
143	static void **server_completed_job_p_array = NULL;
144	static job_status_s *server_job_status_array = NULL;
145
146	/* Prototypes for internal functions */
147	static int server_initialize(
148	PINT_server_status_flag *server_status_flag,
149	job_status_s *job_status_structs);
150	static int server_initialize_subsystems(
151	PINT_server_status_flag *server_status_flag);
152	static int server_setup_signal_handlers(void);
153	static int server_purge_unexpected_recv_machines(void);
154	static int server_setup_process_environment(int background);
155	static int server_shutdown(
156	PINT_server_status_flag status,
157	int ret, int sig);
158	static void reload_config(void);
159	static void server_sig_handler(int sig);
160	static void hup_sighandler(int sig, siginfo_t info, void secret);
161	static int server_parse_cmd_line_args(int argc, char **argv);
162	#ifdef __PVFS2_SEGV_BACKTRACE__
163	static void bt_sighandler(int sig, siginfo_t info, void secret);
164	#endif
165	static int create_pidfile(char *pidfile);
166	static void write_pidfile(int fd);
167	static void remove_pidfile(void);
168	static int generate_shm_key_hint(int* server_index);
169
170	static void precreate_pool_finalize(void);
171	static int precreate_pool_initialize(int server_index);
172
173	static int precreate_pool_setup_server(const char* host, PVFS_ds_type type,
174	PVFS_fs_id fsid, PVFS_handle* pool_handle);
175	static int precreate_pool_launch_refiller(const char* host, PVFS_ds_type type,
176	PVFS_BMI_addr_t addr, PVFS_fs_id fsid, PVFS_handle pool_handle);
177	static int precreate_pool_count(
178	PVFS_fs_id fsid, PVFS_handle pool_handle, int* count);
179
180	static TROVE_method_id trove_coll_to_method_callback(TROVE_coll_id);
181
182
183	struct server_configuration_s *PINT_get_server_config(void)
184	{
185	return &server_config;
186	}
187
188	int main(int argc, char **argv)
189	{
190	int ret = -1, siglevel = 0;
191	struct PINT_smcb *tmp_op = NULL;
192	uint64_t debug_mask = 0;
193
194	#ifdef WITH_MTRACE
195	mtrace();
196	#endif
197
198	/* Passed to server shutdown function */
199	server_status_flag = SERVER_DEFAULT_INIT;
200
201	/* Enable the gossip interface to send out stderr and set an
202	* initial debug mask so that we can output errors at startup.
203	*/
204	gossip_enable_stderr();
205	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG);
206
207	server_status_flag \|= SERVER_GOSSIP_INIT;
208
209	/* Determine initial server configuration, looking at both command
210	* line arguments and the configuration file.
211	*/
212	ret = server_parse_cmd_line_args(argc, argv);
213	if (ret == PVFS2_VERSION_REQUEST)
214	{
215	return 0;
216	}
217	else if (ret == PVFS2_HELP)
218	{
219	return 0;
220	}
221	else if (ret != 0)
222	{
223	goto server_shutdown;
224	}
225
226	gossip_debug_fp(stderr, 'S', GOSSIP_LOGSTAMP_DATETIME,
227	"PVFS2 Server on node %s version %s starting...\n",
228	s_server_options.server_alias, PVFS2_VERSION);
229
230	/* code to handle older two config file format */
231	ret = PINT_parse_config(&server_config, fs_conf, s_server_options.server_alias);
232	if (ret)
233	{
234	gossip_err("Error: Please check your config files.\n");
235	gossip_err("Error: Server aborting.\n");
236	ret = -PVFS_EINVAL;
237	goto server_shutdown;
238	}
239
240	server_status_flag \|= SERVER_CONFIG_INIT;
241
242	if (!PINT_config_is_valid_configuration(&server_config))
243	{
244	gossip_err("Error: Invalid configuration; aborting.\n");
245	ret = -PVFS_EINVAL;
246	goto server_shutdown;
247	}
248
249	/* reset gossip debug mask based on configuration settings */
250	debug_mask = PVFS_debug_eventlog_to_mask(server_config.event_logging);
251	gossip_set_debug_mask(1, debug_mask);
252	gossip_set_logstamp(server_config.logstamp_type);
253	gossip_debug(GOSSIP_SERVER_DEBUG,"Logging %s (mask %llu)\n",
254	server_config.event_logging, llu(debug_mask));
255
256	/* remove storage space and exit if requested */
257	if (s_server_options.server_remove_storage_space)
258	{
259	ret = PINT_config_pvfs2_rmspace(&server_config);
260	if(ret < 0)
261	{
262	goto server_shutdown;
263	}
264	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG);
265	gossip_debug(GOSSIP_SERVER_DEBUG, "PVFS2 Server: storage space removed. Exiting.\n");
266	gossip_set_debug_mask(1, debug_mask);
267	return(0);
268	}
269
270	/* create storage space and exit if requested */
271	if (s_server_options.server_create_storage_space)
272	{
273	ret = PINT_config_pvfs2_mkspace(&server_config);
274	if(ret < 0)
275	{
276	goto server_shutdown;
277	}
278	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG);
279	gossip_debug(GOSSIP_SERVER_DEBUG, "PVFS2 Server: storage space created. Exiting.\n");
280	gossip_set_debug_mask(1, debug_mask);
281	return(0);
282	}
283
284	server_job_id_array = (job_id_t *)
285	malloc(PVFS_SERVER_TEST_COUNT * sizeof(job_id_t));
286	server_completed_job_p_array = (void **)
287	malloc(PVFS_SERVER_TEST_COUNT * sizeof(void *));
288	server_job_status_array = (job_status_s *)
289	malloc(PVFS_SERVER_TEST_COUNT * sizeof(job_status_s));
290
291	if (!server_job_id_array \|\|
292	!server_completed_job_p_array \|\|
293	!server_job_status_array)
294	{
295	if (server_job_id_array)
296	{
297	free(server_job_id_array);
298	server_job_id_array = NULL;
299	}
300	if (server_completed_job_p_array)
301	{
302	free(server_completed_job_p_array);
303	server_completed_job_p_array = NULL;
304	}
305	if (server_job_status_array)
306	{
307	free(server_job_status_array);
308	server_job_status_array = NULL;
309	}
310	gossip_err("Error: failed to allocate arrays for "
311	"tracking completed jobs.\n");
312	goto server_shutdown;
313	}
314	server_status_flag \|= SERVER_JOB_OBJS_ALLOCATED;
315
316	/* Initialize the server (many many steps) */
317	ret = server_initialize(&server_status_flag, server_job_status_array);
318	if (ret < 0)
319	{
320	gossip_err("Error: Could not initialize server; aborting.\n");
321	goto server_shutdown;
322	}
323
324	#ifndef __PVFS2_DISABLE_PERF_COUNTERS__
325	/* kick off performance update state machine */
326	ret = server_state_machine_alloc_noreq(PVFS_SERV_PERF_UPDATE,
327	&(tmp_op));
328	if (ret == 0)
329	{
330	ret = server_state_machine_start_noreq(tmp_op);
331	}
332	if (ret < 0)
333	{
334	PVFS_perror_gossip("Error: failed to start perf update "
335	"state machine.\n", ret);
336	goto server_shutdown;
337	}
338	#endif
339
340	/* kick off timer for expired jobs */
341	ret = server_state_machine_alloc_noreq(
342	PVFS_SERV_JOB_TIMER, &(tmp_op));
343	if (ret == 0)
344	{
345	ret = server_state_machine_start_noreq(tmp_op);
346	}
347	if (ret < 0)
348	{
349	PVFS_perror_gossip("Error: failed to start job timer "
350	"state machine.\n", ret);
351	goto server_shutdown;
352	}
353
354	gossip_debug_fp(stderr, 'S', GOSSIP_LOGSTAMP_DATETIME,
355	"PVFS2 Server ready.\n");
356
357	/* Initialization complete; process server requests indefinitely. */
358	for ( ;; )
359	{
360	int i, comp_ct = PVFS_SERVER_TEST_COUNT;
361
362	if (signal_recvd_flag != 0)
363	{
364	/* If the signal is a SIGHUP, catch and reload configuration */
365	if (signal_recvd_flag == SIGHUP)
366	{
367	reload_config();
368
369	/* re-open log file to allow normal rotation */
370	gossip_reopen_file(server_config.logfile, "a");
371	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG);
372	gossip_debug(GOSSIP_SERVER_DEBUG,
373	"Re-opened log %s, continuing\n",
374	server_config.logfile);
375	gossip_set_debug_mask(1, debug_mask);
376	signal_recvd_flag = 0; /* Reset the flag */
377	}
378	else
379	{
380	/*
381	* If we received a signal and we have drained all the state
382	* machines that were in progress, we initiate a shutdown of
383	* the server. Find out if we can exit now * by checking if
384	* all s_ops (for expected messages) have either finished or
385	* timed out,
386	*/
387	if (qlist_empty(&inprogress_sop_list))
388	{
389	ret = 0;
390	siglevel = signal_recvd_flag;
391	goto server_shutdown;
392	}
393	/* not completed. continue... */
394	}
395	}
396	ret = job_testcontext(server_job_id_array,
397	&comp_ct,
398	server_completed_job_p_array,
399	server_job_status_array,
400	PVFS2_SERVER_DEFAULT_TIMEOUT_MS,
401	server_job_context);
402	if (ret < 0)
403	{
404	gossip_lerr("pvfs2-server panic; main loop aborting\n");
405	goto server_shutdown;
406	}
407
408	/*
409	Loop through the completed jobs and handle whatever comes
410	next
411	*/
412	for (i = 0; i < comp_ct; i++)
413	{
414	/* int unexpected_msg = 0; */
415	struct PINT_smcb *smcb = server_completed_job_p_array[i];
416
417	/* NOTE: PINT_state_machine_next() is a function that
418	* is shared with the client-side state machine
419	* processing, so it is defined in the src/common
420	* directory.
421	*/
422	ret = PINT_state_machine_continue(
423	smcb, &server_job_status_array[i]);
424
425	if (SM_ACTION_ISERR(ret)) /* ret < 0 */
426	{
427	PVFS_perror_gossip("Error: state machine processing error", ret);
428	ret = 0;
429	}
430
431	/* else ret == SM_ACTION_DEFERED */
432	}
433	}
434
435	server_shutdown:
436	server_shutdown(server_status_flag, ret, siglevel);
437	/* NOTE: the server_shutdown() function does not return; it always ends
438	* by calling exit. This point in the code should never be reached.
439	*/
440	return -1;
441	}
442
443	/*
444	* Manipulate the pid file. Don't bother returning an error in
445	* the write stage, since there's nothing that can be done about it.
446	*/
447	static int create_pidfile(char *pidfile)
448	{
449	return open(pidfile, (O_CREAT \| O_WRONLY \| O_TRUNC), 0644);
450	}
451
452	static void write_pidfile(int fd)
453	{
454	pid_t pid = getpid();
455	char pid_str[16] = {0};
456	int len;
457	int ret;
458
459	snprintf(pid_str, 16, "%d\n", pid);
460	len = strlen(pid_str);
461	ret = write(fd, pid_str, len);
462	if(ret < len)
463	{
464	gossip_err("Error: failed to write pid file.\n");
465	close(fd);
466	remove_pidfile();
467	return;
468	}
469	close(fd);
470	return;
471	}
472
473	static void remove_pidfile(void)
474	{
475	assert(s_server_options.pidfile);
476	unlink(s_server_options.pidfile);
477	}
478
479	/* server_initialize()
480	*
481	* Handles:
482	* - backgrounding, redirecting logging
483	* - initializing all the subsystems (BMI, Trove, etc.)
484	* - setting up the state table used to map new requests to
485	* state machines
486	* - allocating and posting the initial unexpected message jobs
487	* - setting up signal handlers
488	*/
489	static int server_initialize(
490	PINT_server_status_flag *server_status_flag,
491	job_status_s *job_status_structs)
492	{
493	int ret = 0, i = 0;
494	FILE *dummy;
495	uint64_t debug_mask = 0;
496
497	assert(server_config.logfile != NULL);
498
499	if(!strcmp(server_config.logtype, "file"))
500	{
501	dummy = fopen(server_config.logfile, "a");
502	if (dummy == NULL)
503	{
504	int tmp_errno = errno;
505	gossip_err("error opening log file %s\n",
506	server_config.logfile);
507	return -tmp_errno;
508	}
509	fclose(dummy);
510	}
511
512	/* redirect gossip to specified target if backgrounded */
513	if (s_server_options.server_background)
514	{
515	if(!freopen("/dev/null", "r", stdin))
516	gossip_err("Error: failed to reopen stdin.\n");
517	if(!freopen("/dev/null", "w", stdout))
518	gossip_err("Error: failed to reopen stdout.\n");
519	if(!freopen("/dev/null", "w", stderr))
520	gossip_err("Error: failed to reopen stderr.\n");
521
522	if(!strcmp(server_config.logtype, "syslog"))
523	{
524	ret = gossip_enable_syslog(LOG_INFO);
525	}
526	else if(!strcmp(server_config.logtype, "file"))
527	{
528	ret = gossip_enable_file(server_config.logfile, "a");
529	}
530	else
531	{
532	ret = gossip_enable_stderr();
533	}
534
535	if (ret < 0)
536	{
537	gossip_err("error opening log file %s\n",
538	server_config.logfile);
539	return ret;
540	}
541	/* log starting message again so it appears in log file, not just
542	* console
543	*/
544	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG);
545	gossip_debug(GOSSIP_SERVER_DEBUG,
546	"PVFS2 Server version %s starting.\n", PVFS2_VERSION);
547	debug_mask = PVFS_debug_eventlog_to_mask(server_config.event_logging);
548	gossip_set_debug_mask(1, debug_mask);
549	}
550
551	/* handle backgrounding, setting up working directory, and so on. */
552	ret = server_setup_process_environment(
553	s_server_options.server_background);
554	if (ret < 0)
555	{
556	gossip_err("Error: Could not start server; aborting.\n");
557	return ret;
558	}
559
560	/* Initialize the bmi, flow, trove and job interfaces */
561	ret = server_initialize_subsystems(server_status_flag);
562	if (ret < 0)
563	{
564	gossip_err("Error: Could not initialize server subsystems\n");
565	return ret;
566	}
567
568	*server_status_flag \|= SERVER_STATE_MACHINE_INIT;
569
570	/* Post starting set of BMI unexpected msg buffers */
571	for (i = 0; i < server_config.initial_unexpected_requests; i++)
572	{
573	ret = server_post_unexpected_recv(&job_status_structs[i]);
574	if (ret < 0)
575	{
576	gossip_err("Error posting unexpected recv\n");
577	return ret;
578	}
579	}
580
581	*server_status_flag \|= SERVER_BMI_UNEXP_POST_INIT;
582
583	ret = server_setup_signal_handlers();
584
585	*server_status_flag \|= SERVER_SIGNAL_HANDLER_INIT;
586
587	gossip_debug(GOSSIP_SERVER_DEBUG,
588	"Initialization completed successfully.\n");
589
590	return ret;
591	}
592
593	/* server_setup_process_environment()
594	*
595	* performs normal daemon initialization steps
596	*
597	* returns 0 on success, -PVFS_EINVAL on failure (details will be logged to
598	* gossip)
599	*/
600	static int server_setup_process_environment(int background)
601	{
602	pid_t new_pid = 0;
603	int pid_fd = -1;
604
605	/*
606	* Manage a pid file if requested (for init scripts). Create
607	* the file in the parent before the chdir, but let the child
608	* write his pid and delete it when exiting.
609	*/
610	if (s_server_options.pidfile)
611	{
612	pid_fd = create_pidfile(s_server_options.pidfile);
613	if (pid_fd < 0)
614	{
615	gossip_err("Failed to create pid file %s: %s\n",
616	s_server_options.pidfile, strerror(errno));
617	return(-PVFS_EINVAL);
618	}
619	}
620
621	if (chdir("/"))
622	{
623	gossip_err("cannot change working directory to \"/\" "
624	"(errno = %x). aborting.\n", errno);
625	return(-PVFS_EINVAL);
626	}
627
628	umask(0077);
629
630	if (background)
631	{
632	new_pid = fork();
633	if (new_pid < 0)
634	{
635	gossip_err("error in fork() system call (errno = %x). "
636	"aborting.\n", errno);
637	return(-PVFS_EINVAL);
638	}
639	else if (new_pid > 0)
640	{
641	/* exit parent */
642	exit(0);
643	}
644
645	new_pid = setsid();
646	if (new_pid < 0)
647	{
648	gossip_err("error in setsid() system call. aborting.\n");
649	return(-PVFS_EINVAL);
650	}
651	}
652	if (pid_fd >= 0)
653	{
654	/* note: pid_fd closed by write_pidfile() */
655	write_pidfile(pid_fd);
656	atexit(remove_pidfile);
657	}
658	server_controlling_pid = getpid();
659	return 0;
660	}
661
662	/* server_initialize_subsystems()
663	*
664	* This:
665	* - initializes distribution subsystem
666	* - initializes encoding/decoding subsystem
667	* - initializes BMI
668	* - initializes Trove
669	* - finds the collection IDs for all file systems
670	* - gets a context from Trove
671	* - tells Trove what handles are to be used
672	* for each file system (collection)
673	* - initializes the flow subsystem
674	* - initializes the job subsystem
675	* - gets a job context
676	* - initialize the request scheduler
677	*/
678	static int server_initialize_subsystems(
679	PINT_server_status_flag *server_status_flag)
680	{
681	int ret = -PVFS_EINVAL;
682	char *cur_merged_handle_range = NULL;
683	PINT_llist *cur = NULL;
684	struct filesystem_configuration_s *cur_fs;
685	TROVE_context_id trove_context = -1;
686	char buf[16] = {0};
687	PVFS_fs_id orig_fsid=0;
688	PVFS_ds_flags init_flags = 0;
689	int bmi_flags = BMI_INIT_SERVER;
690	int shm_key_hint;
691	int server_index;
692
693	if(server_config.enable_events)
694	{
695	ret = PINT_event_init(PINT_EVENT_TRACE_TAU);
696	if (ret < 0)
697	{
698	gossip_err("Error initializing event interface.\n");
699	return (ret);
700	}
701
702	/* Define the state machine event:
703	* START: (client_id, request_id, rank, handle, op_id)
704	* STOP: ()
705	*/
706	PINT_event_define_event(
707	NULL, "sm", "%d%d%d%llu%d", "", &PINT_sm_event_id);
708
709	*server_status_flag \|= SERVER_EVENT_INIT;
710	}
711
712	/* Initialize distributions */
713	ret = PINT_dist_initialize(0);
714	if (ret < 0)
715	{
716	gossip_err("Error initializing distribution interface.\n");
717	return ret;
718	}
719	*server_status_flag \|= SERVER_DIST_INIT;
720
721	ret = PINT_encode_initialize();
722	if (ret < 0)
723	{
724	gossip_err("PINT_encode_initialize() failed.\n");
725	return ret;
726	}
727
728	*server_status_flag \|= SERVER_ENCODER_INIT;
729
730	gossip_debug(GOSSIP_SERVER_DEBUG,
731	"Passing %s as BMI listen address.\n",
732	server_config.host_id);
733
734	/* does the configuration dictate that we bind to a specific address? */
735	if(server_config.tcp_bind_specific)
736	{
737	bmi_flags \|= BMI_TCP_BIND_SPECIFIC;
738	}
739
740	/* Have bmi automatically increment reference count on addresses any
741	* time a new unexpected message appears. The server will decrement it
742	* once it has completed processing related to that request.
743	*/
744	bmi_flags \|= BMI_AUTO_REF_COUNT;
745
746	ret = BMI_initialize(server_config.bmi_modules,
747	server_config.host_id,
748	bmi_flags);
749	if (ret < 0)
750	{
751	PVFS_perror_gossip("Error: BMI_initialize", ret);
752	return ret;
753	}
754	#ifdef USE_TRUSTED
755	/* Pass the server_config file pointer to the lower
756	* levels for the trusted connections related functions to be
757	* called
758	*/
759	BMI_set_info(0, BMI_TRUSTED_CONNECTION, (void *) &server_config);
760	gossip_debug(GOSSIP_SERVER_DEBUG, "Enabling trusted connections!\n");
761	#endif
762	*server_status_flag \|= SERVER_BMI_INIT;
763
764	ret = trove_collection_setinfo(0, 0, TROVE_DB_CACHE_SIZE_BYTES,
765	&server_config.db_cache_size_bytes);
766	/* this should never fail */
767	assert(ret == 0);
768	ret = trove_collection_setinfo(0, 0, TROVE_MAX_CONCURRENT_IO,
769	&server_config.trove_max_concurrent_io);
770	/* this should never fail */
771	assert(ret == 0);
772
773	/* help trove chose a differentiating shm key if needed for Berkeley DB */
774	shm_key_hint = generate_shm_key_hint(&server_index);
775	gossip_debug(GOSSIP_SERVER_DEBUG, "Server using shm key hint: %d\n", shm_key_hint);
776	ret = trove_collection_setinfo(0, 0, TROVE_SHM_KEY_HINT, &shm_key_hint);
777	assert(ret == 0);
778
779	if(server_config.db_cache_type && (!strcmp(server_config.db_cache_type,
780	"mmap")))
781	{
782	/* set db cache type to mmap rather than sys */
783	init_flags \|= TROVE_DB_CACHE_MMAP;
784	}
785
786	/* Set the buffer size according to configuration file */
787	BMI_set_info(0, BMI_TCP_BUFFER_SEND_SIZE,
788	(void *)&server_config.tcp_buffer_size_send);
789	BMI_set_info(0, BMI_TCP_BUFFER_RECEIVE_SIZE,
790	(void *)&server_config.tcp_buffer_size_receive);
791
792	ret = trove_initialize(
793	server_config.trove_method,
794	trove_coll_to_method_callback,
795	server_config.data_path,
796	server_config.meta_path,
797	init_flags);
798	if (ret < 0)
799	{
800	PVFS_perror_gossip("Error: trove_initialize", ret);
801
802	gossip_err("\n***********************************************\n");
803	gossip_err("Invalid Storage Space: %s or %s\n\n",
804	server_config.data_path, server_config.meta_path);
805	gossip_err("Storage initialization failed. The most "
806	"common reason\nfor this is that the storage space "
807	"has not yet been\ncreated or is located on a "
808	"partition that has not yet\nbeen mounted. "
809	"If you'd like to create the storage space,\n"
810	"re-run this program with a -f option.\n");
811	gossip_err("\n***********************************************\n");
812	return ret;
813	}
814
815	*server_status_flag \|= SERVER_TROVE_INIT;
816
817	ret = PINT_cached_config_initialize();
818	if(ret < 0)
819	{
820	gossip_err("Error initializing cached_config interface.\n");
821	return(ret);
822	}
823
824	/* initialize the flow interface */
825	ret = PINT_flow_initialize(server_config.flow_modules, 0);
826
827	if (ret < 0)
828	{
829	PVFS_perror_gossip("Error: PINT_flow_initialize", ret);
830	return ret;
831	}
832
833	*server_status_flag \|= SERVER_FLOW_INIT;
834
835	cur = server_config.file_systems;
836	while(cur)
837	{
838	cur_fs = PINT_llist_head(cur);
839	if (!cur_fs)
840	{
841	break;
842	}
843
844	ret = PINT_cached_config_handle_load_mapping(cur_fs);
845	if(ret)
846	{
847	PVFS_perror("Error: PINT_handle_load_mapping", ret);
848	return(ret);
849	}
850
851	/*
852	set storage hints if any. if any of these fail, we
853	can't error out since they're just hints. thus, we
854	complain in logging and continue.
855	*/
856	ret = trove_collection_setinfo(
857	cur_fs->coll_id, 0,
858	TROVE_DIRECTIO_THREADS_NUM,
859	(void *)&cur_fs->directio_thread_num);
860	if (ret < 0)
861	{
862	gossip_err("Error setting directio threads num\n");
863	}
864
865	ret = trove_collection_setinfo(
866	cur_fs->coll_id, 0,
867	TROVE_DIRECTIO_OPS_PER_QUEUE,
868	(void *)&cur_fs->directio_ops_per_queue);
869	if (ret < 0)
870	{
871	gossip_err("Error setting directio ops per queue\n");
872	}
873
874	ret = trove_collection_setinfo(
875	cur_fs->coll_id, 0,
876	TROVE_DIRECTIO_TIMEOUT,
877	(void *)&cur_fs->directio_timeout);
878	if (ret < 0)
879	{
880	gossip_err("Error setting directio threads num\n");
881	}
882
883	ret = trove_collection_lookup(
884	cur_fs->trove_method,
885	cur_fs->file_system_name, &(orig_fsid), NULL, NULL);
886
887	if (ret < 0)
888	{
889	gossip_err("Error initializing trove for filesystem %s\n",
890	cur_fs->file_system_name);
891	return ret;
892	}
893
894	if(orig_fsid != cur_fs->coll_id)
895	{
896	gossip_err("Error: configuration file does not match storage collection.\n");
897	gossip_err(" storage file fs_id: %d\n", (int)orig_fsid);
898	gossip_err(" config file fs_id: %d\n", (int)cur_fs->coll_id);
899	gossip_err("Warning: This most likely means that the configuration\n");
900	gossip_err(" files have been regenerated without destroying and\n");
901	gossip_err(" recreating the corresponding storage collection.\n");
902	return(-PVFS_ENODEV);
903	}
904
905	/*
906	* get a range string that combines all handles for both meta
907	* and data ranges specified in the config file.
908	*
909	* the server isn't concerned with what allocation of handles
910	* are meta and which are data at this level, so we lump them
911	* all together and hand them to trove-handle-mgmt.
912	*/
913	cur_merged_handle_range =
914	PINT_config_get_merged_handle_range_str(
915	&server_config, cur_fs);
916
917	/*
918	* error out if we're not configured to house either a meta or
919	* data handle range at all.
920	*/
921	if (!cur_merged_handle_range)
922	{
923	gossip_err("Error: Invalid handle range for host %s "
924	"(alias %s) specified in file system %s\n",
925	server_config.host_id,
926	PINT_config_get_host_alias_ptr(
927	&server_config, server_config.host_id),
928	cur_fs->file_system_name);
929	return -1;
930	}
931	else
932	{
933	ret = trove_open_context(cur_fs->coll_id, &trove_context);
934	if (ret < 0)
935	{
936	gossip_err("Error initializing trove context\n");
937	return ret;
938	}
939
940	/*
941	set storage hints if any. if any of these fail, we
942	can't error out since they're just hints. thus, we
943	complain in logging and continue.
944	*/
945	ret = trove_collection_setinfo(
946	cur_fs->coll_id, trove_context,
947	TROVE_COLLECTION_HANDLE_TIMEOUT,
948	(void *)&cur_fs->handle_recycle_timeout_sec);
949	if (ret < 0)
950	{
951	gossip_err("Error setting handle timeout\n");
952	}
953
954	if (cur_fs->attr_cache_keywords &&
955	cur_fs->attr_cache_size &&
956	cur_fs->attr_cache_max_num_elems)
957	{
958	ret = trove_collection_setinfo(
959	cur_fs->coll_id, trove_context,
960	TROVE_COLLECTION_ATTR_CACHE_KEYWORDS,
961	(void *)cur_fs->attr_cache_keywords);
962	if (ret < 0)
963	{
964	gossip_err("Error setting attr cache keywords\n");
965	}
966	ret = trove_collection_setinfo(
967	cur_fs->coll_id, trove_context,
968	TROVE_COLLECTION_ATTR_CACHE_SIZE,
969	(void *)&cur_fs->attr_cache_size);
970	if (ret < 0)
971	{
972	gossip_err("Error setting attr cache size\n");
973	}
974	ret = trove_collection_setinfo(
975	cur_fs->coll_id, trove_context,
976	TROVE_COLLECTION_ATTR_CACHE_MAX_NUM_ELEMS,
977	(void *)&cur_fs->attr_cache_max_num_elems);
978	if (ret < 0)
979	{
980	gossip_err("Error setting attr cache max num elems\n");
981	}
982	ret = trove_collection_setinfo(
983	cur_fs->coll_id, trove_context,
984	TROVE_COLLECTION_ATTR_CACHE_INITIALIZE,
985	(void *)0);
986	if (ret < 0)
987	{
988	gossip_err("Error initializing the attr cache\n");
989	}
990	}
991
992	/*
993	add configured merged handle range for this host/fs.
994	NOTE: if the attr cache was properly configured above,
995	this next setinfo may have the opportunity to cache
996	a number of attributes on startup during an iterate.
997	*/
998	ret = trove_collection_setinfo(
999	cur_fs->coll_id, trove_context,
1000	TROVE_COLLECTION_HANDLE_RANGES,
1001	(void *)cur_merged_handle_range);
1002	if (ret < 0)
1003	{
1004	gossip_err("Error adding handle range %s to "
1005	"filesystem %s\n",
1006	cur_merged_handle_range,
1007	cur_fs->file_system_name);
1008	return ret;
1009	}
1010
1011	ret = trove_collection_setinfo(
1012	cur_fs->coll_id, trove_context,
1013	TROVE_COLLECTION_COALESCING_HIGH_WATERMARK,
1014	(void *)&cur_fs->coalescing_high_watermark);
1015	if(ret < 0)
1016	{
1017	gossip_err("Error setting coalescing high watermark\n");
1018	return ret;
1019	}
1020
1021	ret = trove_collection_setinfo(
1022	cur_fs->coll_id, trove_context,
1023	TROVE_COLLECTION_COALESCING_LOW_WATERMARK,
1024	(void *)&cur_fs->coalescing_low_watermark);
1025	if(ret < 0)
1026	{
1027	gossip_err("Error setting coalescing low watermark\n");
1028	return ret;
1029	}
1030
1031	ret = trove_collection_setinfo(
1032	cur_fs->coll_id, trove_context,
1033	TROVE_COLLECTION_META_SYNC_MODE,
1034	(void *)&cur_fs->trove_sync_meta);
1035	if(ret < 0)
1036	{
1037	gossip_err("Error setting coalescing low watermark\n");
1038	return ret;
1039	}
1040
1041	ret = trove_collection_setinfo(
1042	cur_fs->coll_id, trove_context,
1043	TROVE_COLLECTION_IMMEDIATE_COMPLETION,
1044	(void *)&cur_fs->immediate_completion);
1045	if(ret < 0)
1046	{
1047	gossip_err("Error setting trove immediate completion\n");
1048	return ret;
1049	}
1050
1051	gossip_debug(GOSSIP_SERVER_DEBUG, "File system %s using "
1052	"handles:\n\t%s\n", cur_fs->file_system_name,
1053	cur_merged_handle_range);
1054
1055	gossip_debug(GOSSIP_SERVER_DEBUG, "Sync on metadata update "
1056	"for %s: %s\n", cur_fs->file_system_name,
1057	((cur_fs->trove_sync_meta == TROVE_SYNC) ?
1058	"yes" : "no"));
1059
1060	gossip_debug(GOSSIP_SERVER_DEBUG, "Sync on I/O data update "
1061	"for %s: %s\n", cur_fs->file_system_name,
1062	((cur_fs->trove_sync_data == TROVE_SYNC) ?
1063	"yes" : "no"));
1064
1065	gossip_debug(GOSSIP_SERVER_DEBUG, "Export options for "
1066	"%s:\n RootSquash %s\n AllSquash %s\n ReadOnly %s\n"
1067	" AnonUID %u\n AnonGID %u\n", cur_fs->file_system_name,
1068	(cur_fs->exp_flags & TROVE_EXP_ROOT_SQUASH) ? "yes" : "no",
1069	(cur_fs->exp_flags & TROVE_EXP_ALL_SQUASH) ? "yes" : "no",
1070	(cur_fs->exp_flags & TROVE_EXP_READ_ONLY) ? "yes" : "no",
1071	cur_fs->exp_anon_uid, cur_fs->exp_anon_gid);
1072
1073	/* format and pass sync mode to the flow implementation */
1074	snprintf(buf, 16, "%d,%d", cur_fs->coll_id,
1075	cur_fs->trove_sync_data);
1076	PINT_flow_setinfo(NULL, FLOWPROTO_DATA_SYNC_MODE, buf);
1077
1078	trove_close_context(cur_fs->coll_id, trove_context);
1079	free(cur_merged_handle_range);
1080	}
1081
1082	cur = PINT_llist_next(cur);
1083	}
1084
1085	*server_status_flag \|= SERVER_CACHED_CONFIG_INIT;
1086
1087	gossip_debug(GOSSIP_SERVER_DEBUG,
1088	"Storage Init Complete (%s)\n", SERVER_STORAGE_MODE);
1089	gossip_debug(GOSSIP_SERVER_DEBUG, "%d filesystem(s) initialized\n",
1090	PINT_llist_count(server_config.file_systems));
1091
1092	/*
1093	* Migrate database if needed
1094	*/
1095	ret = trove_migrate(server_config.trove_method,
1096	server_config.data_path,
1097	server_config.meta_path);
1098	if (ret < 0)
1099	{
1100	gossip_err("trove_migrate failed: ret=%d\n", ret);
1101	return(ret);
1102	}
1103
1104	ret = job_time_mgr_init();
1105	if(ret < 0)
1106	{
1107	PVFS_perror_gossip("Error: job_time_mgr_init", ret);
1108	return(ret);
1109	}
1110
1111	*server_status_flag \|= SERVER_JOB_TIME_MGR_INIT;
1112
1113	/* initialize Job Interface */
1114	ret = job_initialize(0);
1115	if (ret < 0)
1116	{
1117	PVFS_perror_gossip("Error: job_initialize", ret);
1118	return ret;
1119	}
1120
1121	*server_status_flag \|= SERVER_JOB_INIT;
1122
1123	ret = job_open_context(&server_job_context);
1124	if (ret < 0)
1125	{
1126	gossip_err("Error opening job context.\n");
1127	return ret;
1128	}
1129
1130	*server_status_flag \|= SERVER_JOB_CTX_INIT;
1131
1132	ret = PINT_req_sched_initialize();
1133	if (ret < 0)
1134	{
1135	PVFS_perror_gossip("Error: PINT_req_sched_intialize", ret);
1136	return ret;
1137	}
1138	*server_status_flag \|= SERVER_REQ_SCHED_INIT;
1139
1140	#ifndef __PVFS2_DISABLE_PERF_COUNTERS__
1141	PINT_server_pc = PINT_perf_initialize(server_keys);
1142	if(!PINT_server_pc)
1143	{
1144	gossip_err("Error initializing performance counters.\n");
1145	return(ret);
1146	}
1147	*server_status_flag \|= SERVER_PERF_COUNTER_INIT;
1148	#endif
1149
1150	ret = precreate_pool_initialize(server_index);
1151	if (ret < 0)
1152	{
1153	gossip_err("Error initializing precreate pool.\n");
1154	return (ret);
1155	}
1156
1157	*server_status_flag \|= SERVER_PRECREATE_INIT;
1158
1159	return ret;
1160	}
1161
1162	static int server_setup_signal_handlers(void)
1163	{
1164	struct sigaction new_action;
1165	struct sigaction ign_action;
1166	struct sigaction hup_action;
1167	hup_action.sa_sigaction = (void *)hup_sighandler;
1168	sigemptyset (&hup_action.sa_mask);
1169	hup_action.sa_flags = SA_RESTART \| SA_SIGINFO;
1170	#ifdef __PVFS2_SEGV_BACKTRACE__
1171	struct sigaction segv_action;
1172
1173	segv_action.sa_sigaction = (void *)bt_sighandler;
1174	sigemptyset (&segv_action.sa_mask);
1175	segv_action.sa_flags = SA_RESTART \| SA_SIGINFO \| SA_ONESHOT;
1176	#endif
1177
1178	/* Set up the structure to specify the new action. */
1179	new_action.sa_handler = server_sig_handler;
1180	sigemptyset (&new_action.sa_mask);
1181	new_action.sa_flags = 0;
1182
1183	ign_action.sa_handler = SIG_IGN;
1184	sigemptyset (&ign_action.sa_mask);
1185	ign_action.sa_flags = 0;
1186
1187	/* catch these */
1188	sigaction (SIGILL, &new_action, NULL);
1189	sigaction (SIGTERM, &new_action, NULL);
1190	sigaction (SIGHUP, &hup_action, NULL);
1191	sigaction (SIGINT, &new_action, NULL);
1192	sigaction (SIGQUIT, &new_action, NULL);
1193	#ifdef __PVFS2_SEGV_BACKTRACE__
1194	sigaction (SIGSEGV, &segv_action, NULL);
1195	sigaction (SIGABRT, &segv_action, NULL);
1196	#else
1197	sigaction (SIGSEGV, &new_action, NULL);
1198	sigaction (SIGABRT, &new_action, NULL);
1199	#endif
1200
1201	/* ignore these */
1202	sigaction (SIGPIPE, &ign_action, NULL);
1203	sigaction (SIGUSR1, &ign_action, NULL);
1204	sigaction (SIGUSR2, &ign_action, NULL);
1205
1206	return 0;
1207	}
1208
1209	#ifdef __PVFS2_SEGV_BACKTRACE__
1210
1211	#if defined(REG_EIP)
1212	# define REG_INSTRUCTION_POINTER REG_EIP
1213	#elif defined(REG_RIP)
1214	# define REG_INSTRUCTION_POINTER REG_RIP
1215	#else
1216	# error Unknown instruction pointer location for your architecture, configure without --enable-segv-backtrace.
1217	#endif
1218
1219	/* bt_signalhandler()
1220	*
1221	* prints a stack trace from a signal handler; code taken from a Linux
1222	* Journal article
1223	*
1224	* no return value
1225	*/
1226	static void bt_sighandler(int sig, siginfo_t info, void secret)
1227	{
1228	void *trace[16];
1229	char messages = (char )NULL;
1230	int i, trace_size = 0;
1231	ucontext_t uc = (ucontext_t )secret;
1232
1233	/* Do something useful with siginfo_t */
1234	if (sig == SIGSEGV)
1235	{
1236	gossip_err("PVFS2 server: signal %d, faulty address is %p, "
1237	"from %p\n", sig, info->si_addr,
1238	(void*)uc->uc_mcontext.gregs[REG_INSTRUCTION_POINTER]);
1239	}
1240	else
1241	{
1242	gossip_err("PVFS2 server: signal %d\n", sig);
1243	}
1244
1245	trace_size = backtrace(trace, 16);
1246	/* overwrite sigaction with caller's address */
1247	trace[1] = (void *) uc->uc_mcontext.gregs[REG_INSTRUCTION_POINTER];
1248
1249	messages = backtrace_symbols(trace, trace_size);
1250	/* skip first stack frame (points here) */
1251	for (i=1; i<trace_size; ++i)
1252	gossip_err("[bt] %s\n", messages[i]);
1253
1254	signal_recvd_flag = sig;
1255	return;
1256	}
1257	#endif
1258
1259	/* hup_signalhandler()
1260	*
1261	* Reload mutable configuration values. If there are errors, leave server in
1262	* a running state.
1263	*
1264	* NOTE: this _only_ reloads configuration values related to squashing,
1265	* readonly, and trusted settings. It does not allow reloading of arbitrary
1266	* configuration file settings.
1267	*
1268	* no return value
1269	*/
1270	static void hup_sighandler(int sig, siginfo_t info, void secret)
1271	{
1272	uint64_t debug_mask;
1273	int debug_on;
1274
1275	/* Let's make sure this message is printed out */
1276	gossip_get_debug_mask(&debug_on, &debug_mask); /* Need to set back later */
1277	gossip_set_debug_mask(1, GOSSIP_SERVER_DEBUG); /* Make sure debug set */
1278	gossip_debug(GOSSIP_SERVER_DEBUG, "PVFS2 received server: signal %d\n", sig);
1279	gossip_set_debug_mask(debug_on, debug_mask); /* Set to original values */
1280
1281	/* Set the flag so the next server loop picks it up and reloads config */
1282	signal_recvd_flag = sig;
1283	}
1284
1285	static void reload_config(void)
1286	{
1287	struct server_configuration_s sighup_server_config;
1288	struct server_configuration_s *orig_server_config;
1289	PINT_llist *orig_filesystems = NULL;
1290	PINT_llist *hup_filesystems = NULL;
1291	struct filesystem_configuration_s *orig_fs;
1292	struct filesystem_configuration_s *hup_fs;
1293	int tmp_value = 0;
1294	char **tmp_ptr = NULL;
1295	int *tmp_int_ptr = NULL;
1296
1297	gossip_debug(GOSSIP_SERVER_DEBUG, "Reloading configuration %s\n",
1298	fs_conf);
1299	/* We received a SIGHUP. Update configuration in place */
1300	if (PINT_parse_config(&sighup_server_config, fs_conf, s_server_options.server_alias) < 0)
1301	{
1302	gossip_err("Error: Please check your config files.\n");
1303	gossip_err("Error: SIGHUP unable to update configuration.\n");
1304	PINT_config_release(&sighup_server_config); /* Free memory */
1305	}
1306	else /* Successful load of config */
1307	{
1308	/* Get the current server configuration and update global items */
1309	orig_server_config = get_server_config_struct();
1310	if (orig_server_config->event_logging)
1311	{
1312	free(orig_server_config->event_logging);
1313	}
1314
1315	/* Copy the new logging mask into the current server configuration */
1316	orig_server_config->event_logging = strdup(sighup_server_config.event_logging);
1317
1318	/* Reset the debug mask */
1319	gossip_set_debug_mask(1, PVFS_debug_eventlog_to_mask(orig_server_config->event_logging));
1320
1321	orig_filesystems = server_config.file_systems;
1322	/* Loop and update all stored file systems */
1323	while(orig_filesystems)
1324	{
1325	int found_matching_config = 0;
1326
1327	orig_fs = PINT_llist_head(orig_filesystems);
1328	if(!orig_fs)
1329	{
1330	break;
1331	}
1332	hup_filesystems = sighup_server_config.file_systems;
1333
1334	/* Find the matching fs from sighup */
1335	while(hup_filesystems)
1336	{
1337	hup_fs = PINT_llist_head(hup_filesystems);
1338	if ( !hup_fs )
1339	{
1340	break;
1341	}
1342	if( hup_fs->coll_id == orig_fs->coll_id )
1343	{
1344	found_matching_config = 1;
1345	break;
1346	}
1347	hup_filesystems = PINT_llist_head(hup_filesystems);
1348	}
1349	if(!found_matching_config)
1350	{
1351	gossip_err("Error: SIGHUP unable to update configuration"
1352	"Matching configuration not found.\n");
1353	break;
1354	}
1355	/* Update root squashing. Prelude is only place to accesses
1356	* these values, so no need to lock around them. Swap the
1357	* needed pointers so that server config gets new values,
1358	* and the old values get freed up
1359	*/
1360	orig_fs->exp_flags = hup_fs->exp_flags;
1361
1362	tmp_value = orig_fs->root_squash_count;
1363	orig_fs->root_squash_count = hup_fs->root_squash_count;
1364	hup_fs->root_squash_count = tmp_value;
1365
1366	tmp_ptr = orig_fs->root_squash_hosts;
1367	orig_fs->root_squash_hosts = hup_fs->root_squash_hosts;
1368	hup_fs->root_squash_hosts = tmp_ptr;
1369
1370	tmp_int_ptr = orig_fs->root_squash_netmasks;
1371	orig_fs->root_squash_netmasks = hup_fs->root_squash_netmasks;
1372	hup_fs->root_squash_netmasks = tmp_int_ptr;
1373
1374	tmp_value = orig_fs->root_squash_exceptions_count;
1375	orig_fs->root_squash_exceptions_count = hup_fs->root_squash_exceptions_count;
1376	hup_fs->root_squash_exceptions_count = tmp_value;
1377
1378	tmp_ptr = orig_fs->root_squash_exceptions_hosts;
1379	orig_fs->root_squash_exceptions_hosts = hup_fs->root_squash_exceptions_hosts;
1380	hup_fs->root_squash_exceptions_hosts = tmp_ptr;
1381
1382	tmp_int_ptr = orig_fs->root_squash_exceptions_netmasks;
1383	orig_fs->root_squash_exceptions_netmasks = hup_fs->root_squash_exceptions_netmasks;
1384	hup_fs->root_squash_exceptions_netmasks = tmp_int_ptr;
1385
1386	/* Update all squashing. Prelude is only place to accesses
1387	* these values, so no need to lock around them. Swap
1388	* pointers so that server config gets new values, and
1389	* the old values get freed up
1390	*/
1391	tmp_value = orig_fs->all_squash_count;
1392	orig_fs->all_squash_count = hup_fs->all_squash_count;
1393	hup_fs->all_squash_count = tmp_value;
1394
1395	tmp_ptr = orig_fs->all_squash_hosts;
1396	orig_fs->all_squash_hosts = hup_fs->all_squash_hosts;
1397	hup_fs->all_squash_hosts = tmp_ptr;
1398
1399	tmp_int_ptr = orig_fs->all_squash_netmasks;
1400	orig_fs->all_squash_netmasks = hup_fs->all_squash_netmasks;
1401	hup_fs->all_squash_netmasks = tmp_int_ptr;
1402
1403	/* Update read only. Prelude is only place to accesses
1404	* these values, so no need to lock around them. Swap
1405	* pointers so that server config gets new values, and
1406	* the old values get freed up
1407	*/
1408	tmp_value = orig_fs->ro_count;
1409	orig_fs->ro_count = hup_fs->ro_count;
1410	hup_fs->ro_count = tmp_value;
1411
1412	tmp_ptr = orig_fs->ro_hosts;
1413	orig_fs->ro_hosts = hup_fs->ro_hosts;
1414	hup_fs->ro_hosts = tmp_ptr;
1415
1416	tmp_int_ptr = orig_fs->ro_netmasks;
1417	orig_fs->ro_netmasks = hup_fs->ro_netmasks;
1418	hup_fs->ro_netmasks = tmp_int_ptr;
1419
1420	orig_fs->exp_anon_uid = hup_fs->exp_anon_uid;
1421	orig_fs->exp_anon_gid = hup_fs->exp_anon_gid;
1422
1423	orig_filesystems = PINT_llist_next(orig_filesystems);
1424	}
1425	#ifdef USE_TRUSTED
1426	server_config.ports_enabled = sighup_server_config.ports_enabled;
1427	server_config.allowed_ports[0] = sighup_server_config.allowed_ports[0];
1428	server_config.allowed_ports[1] = sighup_server_config.allowed_ports[1];
1429	server_config.network_enabled = sighup_server_config.network_enabled;
1430
1431	tmp_value = server_config.allowed_networks_count;
1432	server_config.allowed_networks_count = sighup_server_config.allowed_networks_count;
1433	sighup_server_config.allowed_networks_count = tmp_value;
1434
1435	tmp_ptr = server_config.allowed_networks;
1436	server_config.allowed_networks = sighup_server_config.allowed_networks;
1437	sighup_server_config.allowed_networks = tmp_ptr;
1438
1439	tmp_int_ptr = server_config.allowed_masks;
1440	server_config.allowed_masks = sighup_server_config.allowed_masks;
1441	sighup_server_config.allowed_masks = tmp_int_ptr;
1442
1443	/* security and security_dtor will be updated in a call
1444	* to BMI_set_info. Need to save old values so they are
1445	* deleted on cleanup
1446	*/
1447	sighup_server_config.security = server_config.security;
1448	sighup_server_config.security_dtor = server_config.security_dtor;
1449
1450	/* The set_info call grabs the interface_mutex, so we are
1451	* basically using that to lock this resource
1452	*/
1453	BMI_set_info(0, BMI_TRUSTED_CONNECTION, (void *) &server_config);
1454	#endif
1455	PINT_config_release(&sighup_server_config); /* Free memory */
1456	}
1457	}
1458
1459	static int server_shutdown(
1460	PINT_server_status_flag status,
1461	int ret, int siglevel)
1462	{
1463	if (siglevel == SIGSEGV)
1464	{
1465	gossip_err("SIGSEGV: skipping cleanup; exit now!\n");
1466	exit(EXIT_FAILURE);
1467	}
1468
1469	gossip_debug(GOSSIP_SERVER_DEBUG,
1470	"* server shutdown in progress *\n");
1471
1472	free(s_server_options.server_alias);
1473
1474	if (status & SERVER_PRECREATE_INIT)
1475	{
1476	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting precreate pool "
1477	" [ ... ]\n");
1478	precreate_pool_finalize();
1479	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] precreate pool "
1480	" [ stopped ]\n");
1481	}
1482
1483	if (status & SERVER_STATE_MACHINE_INIT)
1484	{
1485	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting state machine "
1486	"processor [ ... ]\n");
1487	PINT_state_machine_halt();
1488	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] state machine "
1489	"processor [ stopped ]\n");
1490	}
1491
1492	if (status & SERVER_CACHED_CONFIG_INIT)
1493	{
1494	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting cached "
1495	"config interface [ ... ]\n");
1496	PINT_cached_config_finalize();
1497	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] cached "
1498	"config interface [ stopped ]\n");
1499	}
1500
1501	if (status & SERVER_EVENT_INIT)
1502	{
1503	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting event "
1504	"profiling interface [ ... ]\n");
1505	PINT_event_finalize();
1506	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] event "
1507	"profiling interface [ stopped ]\n");
1508	}
1509
1510	if (status & SERVER_REQ_SCHED_INIT)
1511	{
1512	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting request "
1513	"scheduler [ ... ]\n");
1514	PINT_req_sched_finalize();
1515	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] request "
1516	"scheduler [ stopped ]\n");
1517	}
1518
1519	if (status & SERVER_JOB_CTX_INIT)
1520	{
1521	job_close_context(server_job_context);
1522	}
1523
1524	if (status & SERVER_JOB_INIT)
1525	{
1526	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting job "
1527	"interface [ ... ]\n");
1528	job_finalize();
1529	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] job "
1530	"interface [ stopped ]\n");
1531	}
1532
1533	if (status & SERVER_JOB_TIME_MGR_INIT)
1534	{
1535	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting job time "
1536	"mgr interface [ ... ]\n");
1537	job_time_mgr_finalize();
1538	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] job time "
1539	"mgr interface [ stopped ]\n");
1540	}
1541
1542	if (status & SERVER_FLOW_INIT)
1543	{
1544	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting flow "
1545	"interface [ ... ]\n");
1546	PINT_flow_finalize();
1547	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] flow "
1548	"interface [ stopped ]\n");
1549	}
1550
1551	if (status & SERVER_BMI_INIT)
1552	{
1553	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting bmi "
1554	"interface [ ... ]\n");
1555	BMI_finalize();
1556	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] bmi "
1557	"interface [ stopped ]\n");
1558	}
1559
1560	if (status & SERVER_TROVE_INIT)
1561	{
1562	PINT_llist *cur;
1563	struct filesystem_configuration_s *cur_fs;
1564	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting storage "
1565	"interface [ ... ]\n");
1566
1567	cur = server_config.file_systems;
1568	while(cur)
1569	{
1570	cur_fs = PINT_llist_head(cur);
1571	if (!cur_fs)
1572	{
1573	break;
1574	}
1575	trove_collection_clear(cur_fs->trove_method, cur_fs->coll_id);
1576
1577	cur = PINT_llist_next(cur);
1578	}
1579
1580	trove_finalize(server_config.trove_method);
1581	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] storage "
1582	"interface [ stopped ]\n");
1583	}
1584
1585	if (status & SERVER_ENCODER_INIT)
1586	{
1587	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting encoder "
1588	"interface [ ... ]\n");
1589	PINT_encode_finalize();
1590	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] encoder "
1591	"interface [ stopped ]\n");
1592	}
1593
1594	if (status & SERVER_DIST_INIT)
1595	{
1596	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting dist "
1597	"interface [ ... ]\n");
1598	PINT_dist_finalize();
1599	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] dist "
1600	"interface [ stopped ]\n");
1601	}
1602
1603	if (status & SERVER_PERF_COUNTER_INIT)
1604	{
1605	gossip_debug(GOSSIP_SERVER_DEBUG, "[+] halting performance "
1606	"interface [ ... ]\n");
1607	PINT_perf_finalize(PINT_server_pc);
1608	gossip_debug(GOSSIP_SERVER_DEBUG, "[-] performance "
1609	"interface [ stopped ]\n");
1610	}
1611
1612	if (status & SERVER_GOSSIP_INIT)
1613	{
1614	gossip_debug(GOSSIP_SERVER_DEBUG,
1615	"[*] halting logging interface\n");
1616	gossip_disable();
1617	}
1618
1619	if (status & SERVER_CONFIG_INIT)
1620	{
1621	PINT_config_release(&server_config);
1622	}
1623
1624	if (status & SERVER_JOB_OBJS_ALLOCATED)
1625	{
1626	free(server_job_id_array);
1627	free(server_completed_job_p_array);
1628	free(server_job_status_array);
1629	}
1630
1631	if(siglevel == 0 && ret != 0)
1632	{
1633	exit(EXIT_FAILURE);
1634	}
1635	exit(EXIT_SUCCESS);
1636	}
1637
1638	static void server_sig_handler(int sig)
1639	{
1640	struct sigaction new_action;
1641
1642	if (getpid() == server_controlling_pid)
1643	{
1644	if (sig != SIGSEGV)
1645	{
1646	gossip_err("\nPVFS2 server got signal %d "
1647	"(server_status_flag: %d)\n",
1648	sig, (int)server_status_flag);
1649	}
1650
1651	/* ignore further invocations of this signal */
1652	new_action.sa_handler = SIG_IGN;
1653	sigemptyset(&new_action.sa_mask);
1654	new_action.sa_flags = 0;
1655	sigaction (sig, &new_action, NULL);
1656
1657	/* set the signal_recvd_flag on critical errors to cause the
1658	* server to exit gracefully on the next work cycle
1659	*/
1660	signal_recvd_flag = sig;
1661	/*
1662	* iterate through all the machines that we had posted for
1663	* unexpected BMI messages and deallocate them.
1664	* From now the server will only try and finish operations
1665	* that are already in progress, wait for them to timeout
1666	* or complete before initiating shutdown
1667	*/
1668	server_purge_unexpected_recv_machines();
1669	}
1670	}
1671
1672	static void usage(int argc, char **argv)
1673	{
1674	gossip_err("Usage: %s: [OPTIONS] <global_config_file> "
1675	"\n\n", argv[0]);
1676	gossip_err(" -d, --foreground\t"
1677	"will keep server in the foreground\n");
1678	gossip_err(" -f, --mkfs\t\twill cause server to "
1679	"create file system storage and exit\n");
1680	gossip_err(" -h, --help\t\twill show this message\n");
1681	gossip_err(" -r, --rmfs\t\twill cause server to "
1682	"remove file system storage and exit\n");
1683	gossip_err(" -v, --version\t\toutput version information "
1684	"and exit\n");
1685	gossip_err(" -p, --pidfile <file>\twrite process id to file\n");
1686	gossip_err(" -a, --alias <alias>\tuse the specified alias for this node\n");
1687	}
1688
1689	static int server_parse_cmd_line_args(int argc, char **argv)
1690	{
1691	int ret = 0, option_index = 0;
1692	int total_arguments = 0;
1693	const char *cur_option = NULL;
1694	static struct option long_opts[] =
1695	{
1696	{"foreground",0,0,0},
1697	{"mkfs",0,0,0},
1698	{"help",0,0,0},
1699	{"rmfs",0,0,0},
1700	{"version",0,0,0},
1701	{"pidfile",1,0,0},
1702	{"alias",0,0,0},
1703	{0,0,0,0}
1704	};
1705
1706	while ((ret = getopt_long(argc, argv,"dfhrvp:a:e",
1707	long_opts, &option_index)) != -1)
1708	{
1709	total_arguments++;
1710	switch (ret)
1711	{
1712	case 0:
1713	cur_option = long_opts[option_index].name;
1714	assert(cur_option);
1715
1716	if (strcmp("foreground", cur_option) == 0)
1717	{
1718	goto do_foreground;
1719	}
1720	else if (strcmp("mkfs", cur_option) == 0)
1721	{
1722	goto do_mkfs;
1723	}
1724	else if (strcmp("help", cur_option) == 0)
1725	{
1726	goto do_help;
1727	}
1728	else if (strcmp("rmfs", cur_option) == 0)
1729	{
1730	goto do_rmfs;
1731	}
1732	else if (strcmp("version", cur_option) == 0)
1733	{
1734	goto do_version;
1735	}
1736	else if (strcmp("pidfile", cur_option) == 0)
1737	{
1738	goto do_pidfile;
1739	}
1740	else if (strcmp("alias", cur_option) == 0)
1741	{
1742	goto do_alias;
1743	}
1744	break;
1745	case 'v':
1746	do_version:
1747	printf("%s (mode: %s)\n", PVFS2_VERSION,
1748	SERVER_STORAGE_MODE);
1749	return PVFS2_VERSION_REQUEST;
1750	case 'r':
1751	do_rmfs:
1752	s_server_options.server_remove_storage_space = 1;
1753	case 'f':
1754	do_mkfs:
1755	s_server_options.server_create_storage_space = 1;
1756	break;
1757	case 'd':
1758	do_foreground:
1759	s_server_options.server_background = 0;
1760	break;
1761	case 'p':
1762	do_pidfile:
1763	total_arguments++;
1764	s_server_options.pidfile = optarg;
1765	if(optarg[0] != '/')
1766	{
1767	gossip_err("Error: pidfile must be specified with an absolute path.\n");
1768	goto parse_cmd_line_args_failure;
1769	}
1770	break;
1771	case 'a':
1772	do_alias:
1773	total_arguments++;
1774	s_server_options.server_alias = strdup(optarg);
1775	break;
1776	case '?':
1777	case 'h':
1778	do_help:
1779	usage(argc, argv);
1780	if(s_server_options.server_alias)
1781	{
1782	free(s_server_options.server_alias);
1783	}
1784	return PVFS2_HELP;
1785	default:
1786	parse_cmd_line_args_failure:
1787	usage(argc, argv);
1788	if(s_server_options.server_alias)
1789	{
1790	free(s_server_options.server_alias);
1791	}
1792	return 1;
1793	}
1794	}
1795
1796	if(argc < optind)
1797	{
1798	gossip_err("Missing config file in command line arguments\n");
1799	goto parse_cmd_line_args_failure;
1800	}
1801
1802	if (argv[optind][0] != '/')
1803	{
1804	if( getcwd(startup_cwd, PATH_MAX) < 0 )
1805	{
1806	gossip_err("Failed to get current working directory to create "
1807	"absolute path for configuration file: %s\n",
1808	strerror(errno));
1809	}
1810
1811	if( (strlen(argv[optind]) + strlen(startup_cwd) + 1) >= PATH_MAX )
1812	{
1813	gossip_err("Config file path greater than %d characters\n",
1814	PATH_MAX);
1815	goto parse_cmd_line_args_failure;
1816	}
1817
1818	if( strncat(startup_cwd, "/", PATH_MAX) == NULL )
1819	{
1820	gossip_err("Failure creating absolute path from relative "
1821	"configuration file path\n");
1822	goto parse_cmd_line_args_failure;
1823	}
1824
1825	/* copy the relative path into the string for the user */
1826	if( strncat(startup_cwd, argv[optind++], PATH_MAX) == NULL )
1827	{
1828	gossip_err("Failure creating absolute path from relative "
1829	"configuration file path\n");
1830	goto parse_cmd_line_args_failure;
1831	}
1832
1833	if( strncpy(fs_conf, startup_cwd, PATH_MAX) == NULL )
1834	{
1835	gossip_err("Failure copying created full path into configuration "
1836	"file path\n");
1837	goto parse_cmd_line_args_failure;
1838	}
1839	}
1840	else
1841	{
1842	if( strlen(argv[optind]) >= PATH_MAX )
1843	{
1844	gossip_err("Config file path greater than %d characters\n",
1845	PATH_MAX);
1846	goto parse_cmd_line_args_failure;
1847	}
1848	if( strncpy( fs_conf, argv[optind++], PATH_MAX) == NULL )
1849	{
1850	gossip_err("Failure copying configuration file path\n");
1851	goto parse_cmd_line_args_failure;
1852	}
1853	}
1854
1855	if( fs_conf == NULL )
1856	{
1857	gossip_err("Failure copying configuration file path\n");
1858	goto parse_cmd_line_args_failure;
1859	}
1860
1861	if(argc - total_arguments > 2)
1862	{
1863	/* Assume user is passing in a server.conf. Bit of a hack here to
1864	* support server.conf files in the old format by appending the
1865	* server.conf options onto the fs.conf.
1866	*/
1867	gossip_err("The two config file format is no longer supported. "
1868	"Generate a single fs.conf that uses the new format with the "
1869	"pvfs2-config-convert script.\n\n");
1870	goto parse_cmd_line_args_failure;
1871	}
1872
1873	if (s_server_options.server_alias == NULL)
1874	{
1875	/* Try to guess the alias from the hostname */
1876	s_server_options.server_alias = PINT_util_guess_alias();
1877	}
1878	return 0;
1879	}
1880
1881	/* server_post_unexpected_recv()
1882	*
1883	* Allocates space for an unexpected BMI message and posts this.
1884	*
1885	* Returns 0 on success, -PVFS_error on failure.
1886	*/
1887	int server_post_unexpected_recv(job_status_s *js_p)
1888	{
1889	int ret = -PVFS_EINVAL;
1890	/* job_id_t j_id; */
1891	struct PINT_smcb *smcb = NULL;
1892	struct PINT_server_op *s_op;
1893
1894	gossip_debug(GOSSIP_SERVER_DEBUG,
1895	"server_post_unexpected_recv\n");
1896
1897	if (js_p)
1898	{
1899	ret = PINT_smcb_alloc(&smcb, BMI_UNEXPECTED_OP,
1900	sizeof(struct PINT_server_op),
1901	server_op_state_get_machine,
1902	server_state_machine_terminate,
1903	server_job_context);
1904	if (ret < 0)
1905	{
1906	gossip_lerr("Error: failed to allocate SMCB "
1907	"of op type %x\n", BMI_UNEXPECTED_OP);
1908	return ret;
1909	}
1910	s_op = (struct PINT_server_op *)PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
1911	memset(s_op, 0, sizeof(PINT_server_op));
1912	s_op->op = BMI_UNEXPECTED_OP;
1913	s_op->target_handle = PVFS_HANDLE_NULL;
1914	s_op->target_fs_id = PVFS_FS_ID_NULL;
1915	/* Add an unexpected s_ops to the list */
1916	qlist_add_tail(&s_op->next, &posted_sop_list);
1917
1918	ret = PINT_state_machine_start(smcb, js_p);
1919	if(ret == SM_ACTION_TERMINATE)
1920	{
1921	/* error posting unexpected */
1922	PINT_smcb_free(smcb);
1923	return js_p->error_code;
1924	}
1925	}
1926	return ret;
1927	}
1928
1929	/* server_purge_unexpected_recv_machines()
1930	*
1931	* removes any s_ops that were posted to field unexpected BMI messages
1932	*
1933	* returns 0 on success and -PVFS_errno on failure.
1934	*/
1935	static int server_purge_unexpected_recv_machines(void)
1936	{
1937	struct qlist_head tmp = NULL, tmp2 = NULL;
1938
1939	if (qlist_empty(&posted_sop_list))
1940	{
1941	gossip_err("WARNING: Found empty posted operation list!\n");
1942	return -PVFS_EINVAL;
1943	}
1944	qlist_for_each_safe (tmp, tmp2, &posted_sop_list)
1945	{
1946	PINT_server_op *s_op = qlist_entry(tmp, PINT_server_op, next);
1947
1948	/* Remove s_op from the posted_sop_list */
1949	/* don't see a reason to remove this */
1950	/* will be removed in state machine */
1951	/* if and when message completes after cancellation */
1952	/* qlist_del(&s_op->next); */
1953
1954	/* mark the message for cancellation */
1955	s_op->op_cancelled = 1;
1956
1957	/* cancel the pending job_bmi_unexp operation */
1958	job_bmi_unexp_cancel(s_op->unexp_id);
1959	}
1960	return 0;
1961	}
1962
1963	/* server_state_machine_start()
1964	*
1965	* initializes fields in the s_op structure and begins execution of
1966	* the appropriate state machine
1967	*
1968	* returns 0 on success, -PVFS_errno on failure
1969	*/
1970	int server_state_machine_start(
1971	PINT_smcb *smcb,
1972	job_status_s *js_p)
1973	{
1974	PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
1975	int ret = -PVFS_EINVAL;
1976	PVFS_id_gen_t tmp_id;
1977
1978	gossip_debug(GOSSIP_SERVER_DEBUG,
1979	"server_state_machine_start %p\n",smcb);
1980
1981	ret = PINT_decode(s_op->unexp_bmi_buff.buffer,
1982	PINT_DECODE_REQ,
1983	&s_op->decoded,
1984	s_op->unexp_bmi_buff.addr,
1985	s_op->unexp_bmi_buff.size);
1986
1987	/* acknowledge that the unexpected buffer has been used up.
1988	* If someone decides to do in-place decoding, then we will have to move
1989	* this back to state_machine_complete().
1990	*/
1991	if (ret == -PVFS_EPROTONOSUPPORT)
1992	{
1993	/* we have a protocol mismatch of some sort; try to trigger a
1994	* response that gives a helpful error on client side even
1995	* though we can't interpret what the client was asking for
1996	*/
1997	ret = PINT_smcb_set_op(smcb, PVFS_SERV_PROTO_ERROR);
1998	}
1999	else if (ret == 0)
2000	{
2001	s_op->req = (struct PVFS_server_req *)s_op->decoded.buffer;
2002	ret = PINT_smcb_set_op(smcb, s_op->req->op);
2003	s_op->op = s_op->req->op;
2004	PVFS_hint_add(&s_op->req->hints, PVFS_HINT_SERVER_ID_NAME, sizeof(uint32_t), &server_config.host_index);
2005	PVFS_hint_add(&s_op->req->hints, PVFS_HINT_OP_ID_NAME, sizeof(uint32_t), &s_op->req->op);
2006	}
2007	else
2008	{
2009	PVFS_perror_gossip("Error: PINT_decode failure", ret);
2010	return ret;
2011	}
2012	/* Remove s_op from posted_sop_list and move it to the inprogress_sop_list */
2013	qlist_del(&s_op->next);
2014	qlist_add_tail(&s_op->next, &inprogress_sop_list);
2015
2016	/* set timestamp on the beginning of this state machine */
2017	id_gen_fast_register(&tmp_id, s_op);
2018
2019	if(s_op->req)
2020	{
2021	gossip_debug(GOSSIP_SERVER_DEBUG, "client:%d, reqid:%d, rank:%d\n",
2022	PINT_HINT_GET_CLIENT_ID(s_op->req->hints),
2023	PINT_HINT_GET_REQUEST_ID(s_op->req->hints),
2024	PINT_HINT_GET_RANK(s_op->req->hints));
2025	PINT_EVENT_START(PINT_sm_event_id, server_controlling_pid,
2026	NULL, &s_op->event_id,
2027	PINT_HINT_GET_CLIENT_ID(s_op->req->hints),
2028	PINT_HINT_GET_REQUEST_ID(s_op->req->hints),
2029	PINT_HINT_GET_RANK(s_op->req->hints),
2030	PINT_HINT_GET_HANDLE(s_op->req->hints),
2031	s_op->req->op);
2032	s_op->resp.op = s_op->req->op;
2033	}
2034
2035	s_op->addr = s_op->unexp_bmi_buff.addr;
2036	s_op->tag = s_op->unexp_bmi_buff.tag;
2037
2038	if (!ret)
2039	{
2040	gossip_err("Error: server does not implement request type: %d\n",
2041	(int)s_op->req->op);
2042	PINT_decode_release(&(s_op->decoded),PINT_DECODE_REQ);
2043	return -PVFS_ENOSYS;
2044	}
2045
2046	return PINT_state_machine_invoke(smcb, js_p);
2047	}
2048
2049	/* server_state_machine_alloc_noreq()
2050	*
2051	* allocates and initializes a server state machine that can later be
2052	* started with server_state_machine_start_noreq()
2053	*
2054	* returns 0 on success, -PVFS_error on failure
2055	*/
2056	int server_state_machine_alloc_noreq(
2057	enum PVFS_server_op op,
2058	struct PINT_smcb **new_op)
2059	{
2060	int ret = -PVFS_EINVAL;
2061
2062	gossip_debug(GOSSIP_SERVER_DEBUG,
2063	"server_state_machine_alloc_noreq %d\n",op);
2064
2065	if (new_op)
2066	{
2067	PINT_server_op *tmp_op;
2068	ret = PINT_smcb_alloc(new_op, op,
2069	sizeof(struct PINT_server_op),
2070	server_op_state_get_machine,
2071	server_state_machine_terminate,
2072	server_job_context);
2073	if (ret < 0)
2074	{
2075	gossip_lerr("Error: failed to allocate SMCB "
2076	"of op type %x\n", op);
2077	return ret;
2078	}
2079	tmp_op = PINT_sm_frame(*new_op, PINT_FRAME_CURRENT);
2080	tmp_op->op = op;
2081	tmp_op->target_handle = PVFS_HANDLE_NULL;
2082	tmp_op->target_fs_id = PVFS_FS_ID_NULL;
2083
2084	/* NOTE: We do not add these state machines to the
2085	* in-progress or posted sop lists
2086	*/
2087
2088	ret = 0;
2089	}
2090	return ret;
2091	}
2092
2093	/* server_state_machine_start_noreq()
2094	*
2095	* similar in purpose to server_state_machine_start(), except that it
2096	* kicks off a state machine instance without first receiving a client
2097	* side request
2098	*
2099	* PINT_server_op structure must have been previously allocated using
2100	* server_state_machine_alloc_noreq().
2101	*
2102	* returns 0 on success, -PVFS_error on failure
2103	*/
2104	int server_state_machine_start_noreq(struct PINT_smcb *smcb)
2105	{
2106	struct PINT_server_op *new_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
2107	int ret = -PVFS_EINVAL;
2108	job_status_s tmp_status;
2109
2110	gossip_debug(GOSSIP_SERVER_DEBUG,
2111	"server_state_machine_start_noreq %p\n",smcb);
2112
2113	tmp_status.error_code = 0;
2114
2115	if (new_op)
2116	{
2117
2118	/* add to list of state machines started without a request */
2119	qlist_add_tail(&new_op->next, &noreq_sop_list);
2120
2121	/* execute first state */
2122	ret = PINT_state_machine_start(smcb, &tmp_status);
2123	if (ret < 0)
2124	{
2125	gossip_lerr("Error: failed to start state machine.\n");
2126	return ret;
2127	}
2128	}
2129	return ret;
2130	}
2131
2132	/* server_state_machine_complete()
2133	*
2134	* function to be called at the completion of state machine execution;
2135	* it frees up any resources associated with the state machine that were
2136	* allocated before the state machine started executing. Also returns
2137	* appropriate return value to make the state machine stop transitioning
2138	*
2139	* returns 0
2140	*/
2141	int server_state_machine_complete(PINT_smcb *smcb)
2142	{
2143	PINT_server_op *s_op = PINT_sm_frame(smcb, PINT_FRAME_CURRENT);
2144	PVFS_id_gen_t tmp_id;
2145
2146	gossip_debug(GOSSIP_SERVER_DEBUG,
2147	"server_state_machine_complete %p\n",smcb);
2148
2149	/* set a timestamp on the completion of the state machine */
2150	id_gen_fast_register(&tmp_id, s_op);
2151
2152	if(s_op->req)
2153	{
2154	PINT_EVENT_END(PINT_sm_event_id, server_controlling_pid,
2155	NULL, s_op->event_id, 0);
2156	}
2157
2158	/* release the decoding of the unexpected request */
2159	if (ENCODING_IS_VALID(s_op->decoded.enc_type))
2160	{
2161	PVFS_hint_free(s_op->decoded.stub_dec.req.hints);
2162
2163	PINT_decode_release(&(s_op->decoded),PINT_DECODE_REQ);
2164	}
2165
2166	gossip_ldebug(GOSSIP_BMI_DEBUG_TCP,"server_state_machine_complete: smcb op code (%d).\n"
2167	,s_op->op);
2168	gossip_ldebug(GOSSIP_BMI_DEBUG_TCP,"server_state_machine_complete: "
2169	"s_op->unexp_bmi_buff.buffer (%p) "
2170	"\tNULL(%s).\n"
2171	,s_op->unexp_bmi_buff.buffer
2172	,s_op->unexp_bmi_buff.buffer ? "NO" : "YES");
2173
2174	/* BMI_unexpected_free MUST execute BEFORE BMI_set_info, because BMI_set_info will */
2175	/* remove the addr info from the cur_ref_list if BMI_DEC_ADDR_REF causes the ref */
2176	/* count to become zero. The addr info holds the "unexpected-free" function */
2177	/* pointer. */
2178	BMI_unexpected_free(s_op->unexp_bmi_buff.addr,
2179	s_op->unexp_bmi_buff.buffer);
2180	BMI_set_info(s_op->unexp_bmi_buff.addr, BMI_DEC_ADDR_REF, NULL);
2181	s_op->unexp_bmi_buff.buffer = NULL;
2182
2183
2184	/* Remove s_op from the inprogress_sop_list */
2185	qlist_del(&s_op->next);
2186
2187	return SM_ACTION_TERMINATE;
2188	}
2189
2190	int server_state_machine_terminate(
2191	struct PINT_smcb smcb, job_status_s js_p)
2192	{
2193	/* free the operation structure itself */
2194	gossip_debug(GOSSIP_SERVER_DEBUG,
2195	"server_state_machine_terminate %p\n",smcb);
2196	PINT_smcb_free(smcb);
2197	return SM_ACTION_TERMINATE;
2198	}
2199
2200	struct server_configuration_s *get_server_config_struct(void)
2201	{
2202	return &server_config;
2203	}
2204
2205	/* server_op_get_machine()
2206	*
2207	* looks up the state machine for the op * given and returns it, or
2208	* NULL of the op is out of range.
2209	* pointer to this function set in the control block of server state
2210	* machines.
2211	*/
2212	struct PINT_state_machine_s *server_op_state_get_machine(int op)
2213	{
2214	gossip_debug(GOSSIP_SERVER_DEBUG,
2215	"server_op_state_get_machine %d\n",op);
2216
2217	switch (op)
2218	{
2219	case BMI_UNEXPECTED_OP :
2220	{
2221	return &pvfs2_unexpected_sm;
2222	break;
2223	}
2224	default :
2225	{
2226	if (op >= 0 && op < PVFS_SERV_NUM_OPS)
2227	return PINT_server_req_table[op].params->state_machine;
2228	else
2229	return NULL;
2230	break;
2231	}
2232	}
2233	}
2234
2235	static TROVE_method_id trove_coll_to_method_callback(TROVE_coll_id coll_id)
2236	{
2237	struct filesystem_configuration_s * fs_config;
2238
2239	fs_config = PINT_config_find_fs_id(&server_config, coll_id);
2240	if(!fs_config)
2241	{
2242	return server_config.trove_method;
2243	}
2244	return fs_config->trove_method;
2245	}
2246
2247	#ifndef GOSSIP_DISABLE_DEBUG
2248	void PINT_server_access_debug(PINT_server_op * s_op,
2249	int64_t debug_mask,
2250	const char * format,
2251	...)
2252	{
2253	static char pint_access_buffer[GOSSIP_BUF_SIZE];
2254	struct passwd* pw;
2255	struct group* gr;
2256	va_list ap;
2257
2258	if ((gossip_debug_on) &&
2259	(gossip_debug_mask & debug_mask) &&
2260	(gossip_facility))
2261	{
2262	va_start(ap, format);
2263
2264	pw = getpwuid(s_op->req->credentials.uid);
2265	gr = getgrgid(s_op->req->credentials.gid);
2266	snprintf(pint_access_buffer, GOSSIP_BUF_SIZE,
2267	"%s.%s@%s H=%llu S=%p: %s: %s",
2268	((pw) ? pw->pw_name : "UNKNOWN"),
2269	((gr) ? gr->gr_name : "UNKNOWN"),
2270	BMI_addr_rev_lookup_unexpected(s_op->addr),
2271	llu(s_op->target_handle),
2272	s_op,
2273	PINT_map_server_op_to_string(s_op->req->op),
2274	format);
2275
2276	__gossip_debug_va(debug_mask, 'A', pint_access_buffer, ap);
2277
2278	va_end(ap);
2279	}
2280	}
2281	#endif
2282
2283	/* generate_shm_key_hint()
2284	*
2285	* Makes a best effort to produce a unique shm key (for Trove's Berkeley
2286	* DB use) for each server. By default it will base this on the server's
2287	* position in the fs.conf, but it will fall back to using a random number
2288	*
2289	* returns integer key
2290	*/
2291	static int generate_shm_key_hint(int* server_index)
2292	{
2293	struct host_alias_s *cur_alias = NULL;
2294	struct filesystem_configuration_s *first_fs;
2295
2296	*server_index = 1;
2297
2298	PINT_llist *cur = server_config.host_aliases;
2299
2300	/* iterate through list of aliases in configuration file */
2301	while(cur)
2302	{
2303	cur_alias = PINT_llist_head(cur);
2304	if(!cur_alias)
2305	{
2306	break;
2307	}
2308	if(strcmp(cur_alias->bmi_address, server_config.host_id) == 0)
2309	{
2310	/* match */
2311	/* space the shm keys out by 10 to allow for Berkeley DB using
2312	* using more than one key on each server
2313	*/
2314	first_fs = PINT_llist_head(server_config.file_systems);
2315	return(first_fs->coll_id + (server_index)10);
2316	}
2317
2318	(*server_index)++;
2319	cur = PINT_llist_next(cur);
2320	}
2321
2322	/* If we reach this point, we didn't find this server in the alias list.
2323	* This is not a normal situation, but fall back to using a random
2324	* number for the key just to be safe.
2325	*/
2326	srand((unsigned int)time(NULL));
2327	return(rand());
2328	}
2329
2330	/* precreate_pool_initialize()
2331	*
2332	* starts the infrastructure for managing pools of precreated handles
2333	*
2334	* returns 0 on success, -PVFS_error on failure
2335	*/
2336	static int precreate_pool_initialize(int server_index)
2337	{
2338	PINT_llist *cur_f = server_config.file_systems;
2339	struct filesystem_configuration_s *cur_fs;
2340	int ret = -1;
2341	PVFS_handle pool_handle;
2342	int server_count;
2343	PVFS_BMI_addr_t* addr_array;
2344	const char* host;
2345	int i, j;
2346	int server_type;
2347	int handle_count = 0;
2348	int fs_count = 0;
2349	unsigned int types_to_pool = 0;
2350	struct server_configuration_s *user_opts = get_server_config_struct();
2351	assert(user_opts);
2352
2353	/* iterate through list of file systems */
2354	while(cur_f)
2355	{
2356	cur_fs = PINT_llist_head(cur_f);
2357	if (!cur_fs)
2358	{
2359	break;
2360	}
2361
2362	fs_count++;
2363
2364	/* am I a meta server in this file system? */
2365	ret = PINT_cached_config_check_type(
2366	cur_fs->coll_id,
2367	server_config.host_id,
2368	&server_type);
2369	if(ret < 0)
2370	{
2371	gossip_err("Error: %s not found in configuration file.\n",
2372	server_config.host_id);
2373	gossip_err("Error: configuration file is inconsistent.\n");
2374	return(ret);
2375	}
2376	if(!(server_type & PINT_SERVER_TYPE_META))
2377	{
2378	/* This server is not a meta server for this file system;
2379	* skip doing any precreate setup steps.
2380	*/
2381	cur_f = PINT_llist_next(cur_f);
2382	continue;
2383	}
2384
2385	/* how many servers do we have? */
2386	ret = PINT_cached_config_count_servers(
2387	cur_fs->coll_id, PINT_SERVER_TYPE_ALL, &server_count);
2388	if(ret < 0)
2389	{
2390	gossip_err("Error: unable to count servers for fsid: %d\n",
2391	(int)cur_fs->coll_id);
2392	return(ret);
2393	}
2394
2395	addr_array = malloc(server_count*sizeof(PVFS_BMI_addr_t));
2396	if(!addr_array)
2397	{
2398	gossip_err("Error: unable to allocate book keeping information for "
2399	"precreate pools.\n");
2400	return(-PVFS_ENOMEM);
2401	}
2402
2403	/* resolve addrs for each I/O server */
2404	ret = PINT_cached_config_get_server_array(
2405	cur_fs->coll_id, PINT_SERVER_TYPE_ALL,
2406	addr_array, &server_count);
2407	if(ret < 0)
2408	{
2409	gossip_err("Error: unable retrieve servers for fsid: %d\n",
2410	(int)cur_fs->coll_id);
2411	return(ret);
2412	}
2413
2414	for(i=0; i<server_count; i++)
2415	{
2416	host = PINT_cached_config_map_addr(
2417	cur_fs->coll_id, addr_array[i], &server_type);
2418	if(!strcmp(host, server_config.host_id) == 0)
2419	{
2420	/* this is a peer server */
2421	/* make sure a pool exists for that server,type, fsid pair */
2422
2423	/* set ds type of handles to setup in the server's pool based
2424	* on the server type */
2425	types_to_pool = PVFS_TYPE_NONE;
2426	if( (server_type & PINT_SERVER_TYPE_IO) != 0 )
2427	{
2428	types_to_pool \|= PVFS_TYPE_DATAFILE;
2429	}
2430
2431	if( (server_type & PINT_SERVER_TYPE_META) != 0 )
2432	{
2433	types_to_pool \|= (PVFS_TYPE_METAFILE \| PVFS_TYPE_DIRECTORY \|
2434	PVFS_TYPE_SYMLINK \| PVFS_TYPE_DIRDATA \|
2435	PVFS_TYPE_INTERNAL);
2436	}
2437
2438	/* for each possible bit in the ds_type mask check if we should
2439	* create a pool for it */
2440	for(j = 0; j < PVFS_DS_TYPE_COUNT; j++ )
2441	{
2442	PVFS_ds_type t;
2443	int_to_PVFS_ds_type(j, &t);
2444
2445	/* skip setting up a pool when it doesn't make sense i.e.
2446	* when the remote host doesn't have handle types we want.
2447	* or in the special case that we don't get TYPE_NONE
2448	* handles from IO servers*/
2449	if(((t & types_to_pool) == 0 ) \|\|
2450	((t == PVFS_TYPE_NONE) &&
2451	(server_type == PINT_SERVER_TYPE_IO)) )
2452	{
2453	continue;
2454	}
2455
2456	gossip_debug(GOSSIP_SERVER_DEBUG, "%s: setting up pool on "
2457	"%s, type: %u, fs_id: %llu, handle: %llu\n",
2458	__func__, host, t, llu(cur_fs->coll_id),
2459	llu(pool_handle));
2460	ret = precreate_pool_setup_server(host, t,
2461	cur_fs->coll_id, &pool_handle);
2462	if(ret < 0)
2463	{
2464	gossip_err("Error: precreate_pool_initialize failed to "
2465	"setup pool for %s, type %u\n",
2466	server_config.host_id, t);
2467	return(ret);
2468	}
2469
2470	/* count current handles */
2471	ret = precreate_pool_count(cur_fs->coll_id, pool_handle,
2472	&handle_count);
2473	if(ret < 0)
2474	{
2475	gossip_err("Error: precreate_pool_initialize failed to "
2476	"count pool for %s\n",
2477	server_config.host_id);
2478	return(ret);
2479	}
2480
2481	/* prepare the job interface to use this pool */
2482	ret = job_precreate_pool_register_server(host, t,
2483	cur_fs->coll_id, pool_handle, handle_count,
2484	user_opts->precreate_batch_size);
2485
2486	/* launch sm to take care of refilling */
2487	/* the refiller will only actually launch if the batch count
2488	* for the specified type, t, is greater than 0. Otherwise,
2489	* there is no reason to have a refiller running. */
2490	ret = precreate_pool_launch_refiller(host, t, addr_array[i],
2491	cur_fs->coll_id, pool_handle);
2492	if(ret < 0)
2493	{
2494	gossip_err("Error: precreate_pool_initialize failed to "
2495	"launch refiller SM for %s\n",
2496	server_config.host_id);
2497	return(ret);
2498	}
2499	} // for each PVFS_ds_type
2500	}
2501	}
2502
2503	job_precreate_pool_set_index(server_index);
2504
2505	cur_f = PINT_llist_next(cur_f);
2506	free(addr_array); // local variable, malloc'd above to get BMI addrs
2507
2508	}
2509
2510	return(0);
2511	}
2512
2513	/* precreate_pool_finalize()
2514	*
2515	* shuts down infrastructure for managing pools of precreated handles
2516	*/
2517	static void precreate_pool_finalize(void)
2518	{
2519	/* TODO: anything to do here? */
2520	/* TODO: maybe try to stop pending refiller sms? */
2521	return;
2522	}
2523
2524	/* precreate_pool_setup_server()
2525	*
2526	* This function makes sure that a pool is present for the specified server,
2527	* fsid, and type
2528	*
2529	* host: hostname of server the pool is associated with
2530	* type: DS type of handles to store in the pool
2531	* fsid: fsid of the filesystem the pool is associated with
2532	* handle: out value of the handle of the pool
2533	*
2534	*/
2535	static int precreate_pool_setup_server(const char* host, PVFS_ds_type type,
2536	PVFS_fs_id fsid, PVFS_handle* pool_handle)
2537	{
2538	job_status_s js;
2539	job_id_t job_id;
2540	int ret;
2541	int outcount;
2542	PVFS_handle_extent_array ext_array;
2543
2544	PVFS_ds_keyval key;
2545	PVFS_ds_keyval val;
2546
2547	/* look for the pool handle for this server */
2548
2549	/* the key for the pool must now be server name plus handle type.
2550	* since the key is currently a string it makes some sense to keep
2551	* the whole thing printable instead of just tacking on a PVFS_ds_type
2552	* to the end of the buffer. So, we'll sprint the type as an int and
2553	* tack that on the end. Better that just tacking the bits on?
2554	* Maybe not. */
2555	char type_string[11] = { 0 }; /* 32 bit type only needs 10 digits */
2556	snprintf(type_string, 11, "%u", type);
2557
2558	key.buffer_sz = strlen(host) + strlen(type_string) +
2559	strlen("precreate-pool-") + 2;
2560	key.buffer = malloc(key.buffer_sz);
2561	if(!key.buffer)
2562	{
2563	return(-ENOMEM);
2564	}
2565	snprintf((char*)key.buffer, key.buffer_sz, "precreate-pool-%s-%s",
2566	host, type_string);
2567	key.read_sz = 0;
2568
2569	val.buffer = pool_handle;
2570	val.buffer_sz = sizeof(*pool_handle);
2571	val.read_sz = 0;
2572
2573	ret = job_trove_fs_geteattr(fsid, &key, &val, 0, NULL, 0, &js,
2574	&job_id, server_job_context, NULL);
2575	while(ret == 0)
2576	{
2577	ret = job_test(job_id, &outcount, NULL, &js,
2578	PVFS2_SERVER_DEFAULT_TIMEOUT_MS, server_job_context);
2579	}
2580	if(ret < 0)
2581	{
2582	gossip_err("Error: precreate_pool failed to read fs eattrs.\n");
2583	free(key.buffer);
2584	return(ret);
2585	}
2586	if(js.error_code && js.error_code != -TROVE_ENOENT)
2587	{
2588	gossip_err("Error: precreate_pool failed to read fs eattrs.\n");
2589	free(key.buffer);
2590	return(js.error_code);
2591	}
2592	else if(js.error_code == -TROVE_ENOENT)
2593	{
2594	/* handle doesn't exist yet; let's create it */
2595	gossip_debug(GOSSIP_SERVER_DEBUG, "precreate_pool didn't find handle "
2596	"for %s, type %s; creating now.\n", host, type_string);
2597
2598	/* find extent array for ourselves */
2599	ret = PINT_cached_config_get_server(
2600	fsid, server_config.host_id, PINT_SERVER_TYPE_META, &ext_array);
2601	if(ret < 0)
2602	{
2603	gossip_err("Error: PINT_cached_config_get_meta() failure.\n");
2604	free(key.buffer);
2605	return(ret);
2606	}
2607
2608	/* create a trove object for the pool */
2609	ret = job_trove_dspace_create(fsid, &ext_array, PVFS_TYPE_INTERNAL,
2610	NULL, TROVE_SYNC, NULL, 0, &js, &job_id, server_job_context, NULL);
2611	while(ret == 0)
2612	{
2613	ret = job_test(job_id, &outcount, NULL, &js,
2614	PVFS2_SERVER_DEFAULT_TIMEOUT_MS, server_job_context);
2615	}
2616	if(ret < 0 \|\| js.error_code)
2617	{
2618	gossip_err("Error: precreate_pool failed to create pool.\n");
2619	free(key.buffer);
2620	return(ret < 0 ? ret : js.error_code);
2621	}
2622
2623	*pool_handle = js.handle;
2624
2625	/* store reference to pool handle as collection eattr */
2626	ret = job_trove_fs_seteattr(fsid, &key, &val, TROVE_SYNC, NULL, 0, &js,
2627	&job_id, server_job_context, NULL);
2628	while(ret == 0)
2629	{
2630	ret = job_test(job_id, &outcount, NULL, &js,
2631	PVFS2_SERVER_DEFAULT_TIMEOUT_MS, server_job_context);
2632	}
2633	if(ret < 0 \|\| js.error_code)
2634	{
2635	gossip_err("Error: failed to record precreate pool handle.\n");
2636	gossip_err("Warning: fsck may be needed to recover lost handle.\n");
2637	free(key.buffer);
2638	return(ret < 0 ? ret : js.error_code);
2639	}
2640	gossip_debug(GOSSIP_SERVER_DEBUG, "precreate_pool created handle %llu "
2641	"for %s, type %s.\n", llu(*pool_handle), host,
2642	type_string);
2643
2644	}
2645	else
2646	{
2647	/* handle already exists */
2648	gossip_debug(GOSSIP_SERVER_DEBUG, "precreate_pool found handle %llu "
2649	"for %s, type %s.\n", llu(*pool_handle), host,
2650	type_string);
2651	}
2652	free(key.buffer);
2653	return(0);
2654	}
2655
2656	/* precreate_pool_count()
2657	*
2658	* counts the number of handles stored in a persistent precreate pool
2659	*/
2660	static int precreate_pool_count(
2661	PVFS_fs_id fsid, PVFS_handle pool_handle, int* count)
2662	{
2663	int ret;
2664	job_status_s js;
2665	job_id_t job_id;
2666	int outcount;
2667	PVFS_ds_keyval_handle_info handle_info;
2668
2669	/* try to get the current number of handles from the pool */
2670	ret = job_trove_keyval_get_handle_info(
2671	fsid, pool_handle, TROVE_KEYVAL_HANDLE_COUNT, &handle_info,
2672	NULL, 0, &js, &job_id, server_job_context, NULL);
2673	while(ret == 0)
2674	{
2675	ret = job_test(job_id, &outcount, NULL, &js,
2676	PVFS2_SERVER_DEFAULT_TIMEOUT_MS, server_job_context);
2677	}
2678	if(ret < 0)
2679	{
2680	return(ret);
2681	}
2682
2683	if(js.error_code == -TROVE_ENOENT)
2684	{
2685	/* this really means there aren't any keyvals there yet */
2686	handle_info.count = 0;
2687	}
2688	else if(js.error_code != 0)
2689	{
2690	return(js.error_code);
2691	}
2692
2693	*count = handle_info.count;
2694
2695	return(0);
2696	}
2697
2698	/*
2699	* starts a precreate pool refiller state machine for the specified host and
2700	* type of handle.
2701	* host: the remote host to get handles from
2702	* type: the DS type of handle the refiller will be refilling
2703	* addr: the BMI addr of the remote host
2704	* fsid: the filesystem ID of the fs the pool refiller is associated with
2705	* pool_handle: the handle of the pool itself
2706	*
2707	* This will only be called for a host/type that matches and needs a filler
2708	* so a remote server that is I/O only will only get refillers for datafile
2709	* handles.
2710	*/
2711	static int precreate_pool_launch_refiller(const char* host, PVFS_ds_type type,
2712	PVFS_BMI_addr_t addr, PVFS_fs_id fsid, PVFS_handle pool_handle)
2713	{
2714	struct PINT_smcb *tmp_smcb = NULL;
2715	struct PINT_server_op *s_op;
2716	int ret, index = 0;
2717	struct server_configuration_s *user_opts = get_server_config_struct();
2718
2719	assert(user_opts);
2720	PVFS_ds_type_to_int(type, &index);
2721
2722	if( user_opts->precreate_batch_size[index] == 0 )
2723	{
2724	gossip_debug(GOSSIP_SERVER_DEBUG, "%s: NOT launching refiller for "
2725	"host %s, type %d, pool: %llu, batch_size is 0\n",
2726	__func__, host, type, llu(pool_handle));
2727	return 0;
2728	}
2729
2730	/* allocate smcb */
2731	ret = server_state_machine_alloc_noreq(PVFS_SERV_PRECREATE_POOL_REFILLER,
2732	&(tmp_smcb));
2733	if (ret < 0)
2734	{
2735	return(ret);
2736	}
2737
2738	s_op = PINT_sm_frame(tmp_smcb, PINT_FRAME_CURRENT);
2739	s_op->u.precreate_pool_refiller.host = strdup(host);
2740	if(!s_op->u.precreate_pool_refiller.host)
2741	{
2742	PINT_smcb_free(tmp_smcb);
2743	return(ret);
2744	}
2745
2746	/* set this refillers handle range based on the type of handle it will
2747	* hold. If it's a datafile get an IO server range, otherwise get a meta
2748	* range. */
2749	ret = PINT_cached_config_get_server( fsid, host,
2750	((type == PVFS_TYPE_DATAFILE) ? PINT_SERVER_TYPE_IO :
2751	PINT_SERVER_TYPE_META),
2752	&s_op->u.precreate_pool_refiller.handle_extent_array);
2753	if(ret < 0)
2754	{
2755	free(s_op->u.precreate_pool_refiller.host);
2756	PINT_smcb_free(tmp_smcb);
2757	return(ret);
2758	}
2759
2760	gossip_debug(GOSSIP_SERVER_DEBUG, "%s: launching refiller for host %s, "
2761	"type %d, pool: %llu, batch size %d (index %d)\n", __func__,
2762	s_op->u.precreate_pool_refiller.host, type, llu(pool_handle),
2763	user_opts->precreate_batch_size[index], index);
2764
2765	s_op->u.precreate_pool_refiller.pool_handle = pool_handle;
2766	s_op->u.precreate_pool_refiller.fsid = fsid;
2767	s_op->u.precreate_pool_refiller.type = type;
2768	s_op->u.precreate_pool_refiller.host_addr = addr;
2769
2770	/* start sm */
2771	ret = server_state_machine_start_noreq(tmp_smcb);
2772	if (ret < 0)
2773	{
2774	free(s_op->u.precreate_pool_refiller.host);
2775	PINT_smcb_free(tmp_smcb);
2776	return(ret);
2777	}
2778
2779	return(0);
2780	}
2781
2782	/*
2783	* Local variables:
2784	* c-indent-level: 4
2785	* c-basic-offset: 4
2786	* End:
2787	*
2788	* vim: ts=8 sts=4 sw=4 expandtab
2789	*/

Note: See TracBrowser for help on using the browser.

Context Navigation

root/branches/Orange-Branch/src/server/pvfs2-server.c @ 8863

Download in other formats: