1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#include <sys/types.h>
28#include <sys/wait.h>
29#include <stdio.h>
30#include <errno.h>
31#include <limits.h>
32#include <fcntl.h>
33#include <strings.h>
34#include <stdlib.h>
35#include <unistd.h>
36#include <pthread.h>
37#include <thread.h>
38
39#include <locale.h>
40#include <langinfo.h>
41#include <libintl.h>
42#include <stdarg.h>
43
44#include <sys/nsctl/rdc_io.h>
45#include <sys/nsctl/rdc_ioctl.h>
46#include <sys/nsctl/rdc_prot.h>
47
48#include <sys/nsctl/cfg.h>
49
50#include <sys/unistat/spcs_s.h>
51#include <sys/unistat/spcs_s_u.h>
52#include <sys/unistat/spcs_errors.h>
53
54#include <sys/nsctl/librdc.h>
55
56#include "rdcadm.h"
57
58
59#define	RDCADM "/usr/sbin/sndradm"
60#define	IIADM "/usr/sbin/iiadm"
61
62#define	UPDATE "update"
63#define	NOUPDATE "noupdate"
64
65#define	RESYNC_SLEEP	(3 * 60)	/* Three minutes */
66#define	MAIN_SLEEP	(5 * 60)	/* Five minutes */
67#define	CFG_WAIT_SLEEP	(5)		/* 5 sec */
68
69#define	MAXHOSTS 1024
70mutex_t cfglock = DEFAULTMUTEX;
71#define	LOCKCFG() (void) mutex_lock(&cfglock);
72#define	UNLOCKCFG() (void) mutex_unlock(&cfglock);
73
74typedef struct host_list_s {
75	char *hosts[MAXHOSTS];
76	int numhosts;
77	int configured[MAXHOSTS];
78	mutex_t hosts_mutex;
79} host_list_t;
80
81host_list_t *host_list;
82
83extern char *basename(char *);
84int rdc_maxsets;
85char *program;
86
87static int clustered = 0;
88
89int isnewhost(char *host);
90void *wait_sync_event();
91void *wait_link_down(void *host);
92void rdc_sync(char *tohost);
93void remove_from_hostlist(char *host);
94void sync_start(char *master);
95void sync_complete(char *master);
96void cleanup_hostlist();
97void group_start(char *group);
98void group_complete(char *group);
99
100
101void
102init_host_list(void)
103{
104	host_list = calloc(1, sizeof (host_list_t));
105	if (host_list == NULL) {
106		spcs_log("sndr", NULL,
107		    gettext("host list not initialized, cannot run"));
108		rdc_err(NULL, gettext("host list not initialized, cannot run"));
109	}
110	(void) mutex_init(&host_list->hosts_mutex, USYNC_THREAD, NULL);
111}
112
113/* ARGSUSED */
114#ifdef lint
115void
116sndrsyncd_lintmain(argc, argv)
117#else
118int
119main(argc, argv)
120#endif
121int argc;
122char **argv;
123{
124	rdc_status_t *rdc_info;
125	int size;
126	int i;
127	pid_t pid;
128	spcs_s_info_t ustatus;
129	int rc, trc;
130	int first = 0;
131	char *required;
132
133	(void) setlocale(LC_ALL, "");
134	(void) textdomain("rdc");
135
136	ustatus = spcs_s_ucreate();
137
138	program = basename(argv[0]);
139
140	init_host_list();
141
142	rc = rdc_check_release(&required);
143	if (rc < 0) {
144		rdc_err(NULL,
145		    gettext("unable to determine the current "
146		    "Solaris release: %s\n"), strerror(errno));
147		/* NOTREACHED */
148	} else if (rc == FALSE) {
149		rdc_err(NULL,
150		    gettext("incorrect Solaris release (requires %s)\n"),
151		    required);
152		/* NOTREACHED */
153	}
154
155	clustered = cfg_iscluster();
156	if (clustered < 0) {
157		rdc_err(NULL, gettext("unable to ascertain environment"));
158	}
159
160	rdc_maxsets = rdc_get_maxsets();
161	if (rdc_maxsets == -1) {
162		spcs_log("sndr", NULL,
163		    gettext("%s: unable to get maxsets value from kernel"),
164		    program);
165		rdc_err(NULL,
166		    gettext("unable to get maxsets value from kernel"));
167	}
168	size = sizeof (rdc_status_t) + (sizeof (rdc_set_t) * (rdc_maxsets - 1));
169	rdc_info = malloc(size);
170	if (rdc_info == NULL) {
171		spcs_log("sndr", NULL,
172		    gettext("%s: unable to allocate %ld bytes"),
173		    program, size);
174		rdc_err(NULL,
175			gettext("unable to allocate %ld bytes"), size);
176	}
177	bzero(rdc_info, size);
178
179	rdc_info->nset = rdc_maxsets;
180
181	/*
182	 * Fork off a child that becomes the daemon.
183	 */
184	if ((pid = fork()) > 0)
185		exit(0);
186	else if (pid < 0) {
187		spcs_log("sndr", NULL,
188		    gettext("%s: cannot fork: %s"),
189		    program, strerror(errno));
190		rdc_err(NULL, gettext("cannot fork: %s\n"),
191		    strerror(errno));
192	}
193
194	/*
195	 * In child - become daemon.
196	 */
197
198	for (i = 0; i < 3; i++)
199		(void) close(i);
200
201	(void) open("/dev/console", O_WRONLY|O_APPEND);
202	(void) dup(0);
203	(void) dup(0);
204	(void) close(0);
205
206	(void) setpgrp();
207
208	(void) setlocale(LC_ALL, "");
209	(void) textdomain("rdc");
210
211	/* launch a thread to wait for sync start and sync stop events */
212
213	if ((trc = thr_create(NULL, 0, wait_sync_event, NULL,
214	    THR_BOUND|THR_DETACHED, NULL)) != 0) {
215		spcs_log("sndr", NULL,
216		    gettext("%s: unable to create thread wait_sync_event"),
217		    program);
218		rdc_warn(NULL,
219		    gettext("%s unable to create thread wait_sync_event"),
220		    program);
221	} else {
222#ifdef DEBUG
223		spcs_log("sndr", NULL,
224		    gettext("%s: thread wait_sync_event started"), program);
225#endif
226		;
227	}
228
229	for (;;) {
230		if (!first) {
231			first++;
232			(void) sleep(15);
233		} else
234			(void) sleep(MAIN_SLEEP);
235
236		bzero(rdc_info, size);
237		rdc_info->nset = rdc_maxsets;
238		if (RDC_IOCTL(RDC_STATUS, rdc_info, 0, 0, 0, 0, ustatus)
239		    != SPCS_S_OK) {
240			spcs_log("sndr", &ustatus,
241			    gettext("%s: status ioctl"),
242			    program);
243			rdc_warn(&ustatus, gettext("status ioctl"));
244			continue;
245		}
246
247		cleanup_hostlist(rdc_info); /* remove non-existent hosts */
248
249		/*
250		 * Check all enabled sets to see if a new remote host has
251		 * appeared.
252		 */
253		for (i = 0; i < rdc_maxsets; i++) {
254			if (!(rdc_info->rdc_set[i].flags & RDC_ENABLED))
255				continue;
256			/* spawn a new thread for each new host found */
257			if (isnewhost(rdc_info->rdc_set[i].secondary.intf)) {
258				/*
259				 * right now, we could be here before
260				 * the database did the write for this set
261				 * I could check the lock on the database
262				 * but I am just going to give up some time here
263				 * instead. Why do the allocations etc, etc
264				 * if the set is enabled in the kernel and not
265				 * in the config, we know that this set has the
266				 * lock. Why bother adding more contention to
267				 * the lock.
268				 * this is a daemon, afterall. its got time
269				 */
270				(void) sleep(CFG_WAIT_SLEEP);
271
272				spcs_log("sndr", NULL,
273				    gettext("%s: new host found (%s) starting "
274				    "its autosync thread"), program,
275				    rdc_info->rdc_set[i].secondary.intf);
276
277				trc = thr_create(NULL, 0, wait_link_down,
278				    (void *) rdc_info->rdc_set[i].\
279secondary.intf, THR_BOUND|THR_DETACHED, NULL);
280
281				if (trc != 0) {
282					spcs_log("sndr", NULL,
283					    gettext(
284					    "%s create new autosync "
285					    "thread failed"), program);
286					rdc_warn(NULL, gettext(
287					    "%s create new autosync "
288					    "thread failed"), program);
289				}
290			}
291		}
292	}
293	/* NOTREACHED */
294}
295
296
297/*
298 * The kernel wakes up this function every time it detects the link to the
299 * specified host has dropped.
300 */
301void *
302wait_link_down(void *thehost)
303{
304	char *host = (char *)thehost;
305	char tmphost[MAX_RDC_HOST_SIZE] = { '\0' };
306	spcs_s_info_t ustatus;
307
308	if (host)
309		(void) strncpy(tmphost, host, MAX_RDC_HOST_SIZE);
310
311	ustatus = spcs_s_ucreate();
312
313	/* Never give up */
314	for (;;) {
315#ifdef DEBUG
316		spcs_log("sndr", NULL,
317		    gettext("%s: awaiting link down ioctl for %s"),
318		    program, host[0] == '\0' ? tmphost : host);
319#endif
320		if (RDC_IOCTL(RDC_LINK_DOWN, host, 0, 0, 0, 0, ustatus)
321		    != SPCS_S_OK) {
322			spcs_log("sndr", &ustatus,
323			    gettext("%s: link down ioctl"),
324			    program);
325			rdc_warn(&ustatus, gettext("link down ioctl"));
326			continue;
327		}
328#ifdef DEBUG
329
330		spcs_log("sndr", NULL,
331		    gettext("%s: received link down ioctl for %s"),
332		    program, host[0] == '\0' ? tmphost : host);
333#endif
334		rdc_sync(host[0] == '\0' ? tmphost : host);
335	}
336	/* LINTED */
337}
338
339
340/*
341 * Called when the link to the specified host has dropped.
342 * For all Remote Mirror sets using the link that have autosync on,
343 * issue rdcadm -u commands until they complete successfully.
344 */
345void
346rdc_sync(char *tohost)
347{
348	rdc_set_t *rdc_set = NULL;
349	int *sync_done = NULL;
350	int sets = 0;
351	int syncs_done = 0;
352	char cmd[256];
353	rdc_config_t parms = { 0 };
354	spcs_s_info_t ustatus;
355	int i;
356	int setnumber;
357	int numfound = 0;
358	char buf[CFG_MAX_BUF];
359	char key[CFG_MAX_KEY];
360	CFGFILE *cfg = NULL;
361	int size;
362	int first = 0;
363	int death = 0;
364	int cfglocked = 0;
365
366	ustatus = spcs_s_ucreate();
367
368	size = sizeof (rdc_set_t) * rdc_maxsets;
369	rdc_set = malloc(size);
370	if (rdc_set == NULL) {
371		spcs_log("sndr", NULL,
372		    gettext("%s: unable to allocate %ld bytes"),
373		    program, size);
374		rdc_warn(NULL,
375			gettext("unable to allocate %ld bytes"), size);
376		goto done;
377	}
378	bzero(rdc_set, size);
379	size = sizeof (int) * rdc_maxsets;
380	sync_done = malloc(size);
381	if (sync_done == NULL) {
382		spcs_log("sndr", NULL,
383		    gettext("%s: unable to allocate %ld bytes"),
384		    program, size);
385		rdc_warn(NULL,
386			gettext("unable to allocate %ld bytes"), size);
387		goto done;
388	}
389	bzero(sync_done, size);
390
391	/*
392	 * Get all sndr entries with shost matching tohost, and save the
393	 * details in an array.
394	 */
395	for (i = 0; i < rdc_maxsets; i++) {
396		setnumber = i + 1;
397		bzero(buf, sizeof (buf));
398		bzero(key, sizeof (key));
399
400		(void) snprintf(key, sizeof (key), "sndr.set%d.shost",
401		    setnumber);
402
403		if (!cfglocked) {
404			LOCKCFG();
405			if ((cfg = cfg_open(NULL)) == NULL) {
406				spcs_log("sndr", NULL,
407				    gettext("%s: error opening config"),
408				    program);
409
410				rdc_warn(NULL,
411				    gettext("error opening config"));
412				UNLOCKCFG();
413				goto done;
414			}
415
416			if (!cfg_lock(cfg, CFG_RDLOCK)) {
417				spcs_log("sndr", NULL,
418				    gettext("%s: error locking config"),
419				    program);
420				rdc_warn(NULL, gettext("error locking config"));
421				goto done;
422			}
423		}
424
425		cfglocked = 1;
426
427		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
428			if (numfound == 0) /* no matching hosts */
429				death = 1; /* thread will exit */
430			break;
431		}
432		if (strcmp(buf, tohost) != 0)
433			continue;
434
435		numfound++;
436		(void) strncpy(rdc_set[sets].secondary.intf, buf,
437		    MAX_RDC_HOST_SIZE);
438
439		/* Got a matching entry */
440
441		(void) snprintf(key, sizeof (key), "sndr.set%d.phost",
442		    setnumber);
443		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
444			break;
445		(void) strncpy(rdc_set[sets].primary.intf, buf,
446		    MAX_RDC_HOST_SIZE);
447
448		(void) snprintf(key, sizeof (key), "sndr.set%d.primary",
449		    setnumber);
450		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
451			break;
452		(void) strncpy(rdc_set[sets].primary.file, buf, NSC_MAXPATH);
453
454		(void) snprintf(key, sizeof (key), "sndr.set%d.secondary",
455		    setnumber);
456		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
457			break;
458		(void) strncpy(rdc_set[sets].secondary.file, buf, NSC_MAXPATH);
459
460		parms.command = RDC_CMD_STATUS;
461		bcopy((void *)(&rdc_set[sets]), (void *)(&parms.rdc_set[0]),
462		    sizeof (rdc_set_t));
463
464		/*
465		 * release cfg before diving into the kernel
466		 * this prevents a possible deadlock when doing
467		 * a reverse sync whick will wake up the sync_event
468		 * thread which will try and iiadm -c and hang
469		 * because we still have the cfg_lock. the timed
470		 * wait cv in the kernel will fail the sync and things
471		 * will undeadlock.
472		 */
473
474		cfg_close(cfg);
475		cfg = NULL;
476		cfglocked = 0;
477		UNLOCKCFG();
478
479		if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
480			continue;
481		}
482		if ((parms.rdc_set[0].autosync == 0) ||
483		    (!(parms.rdc_set[0].flags & RDC_LOGGING))) {
484			continue;
485		}
486
487		/* Found a suitable set with autosync on, in logging mode */
488		sets++;
489	}
490
491	if (cfg) {
492		cfg_close(cfg);
493		cfg = NULL;
494		UNLOCKCFG();
495	}
496
497	if (sets == 0) {
498#ifdef DEBUG
499		spcs_log("sndr", NULL,
500		    gettext("%s: no sets requiring autosync found for %s"),
501		    program, tohost);
502#endif
503		if (death) {
504			spcs_log("sndr", NULL,
505			    gettext("%s: autosync thread stopping for %s "
506			    "(host deconfigured)"), program, tohost);
507		}
508		goto done;
509	}
510
511	/* Keep issuing rdcadm -u commands until they have all completed */
512	for (;;) {
513		if (!first)
514			first++;
515		else
516			(void) sleep(RESYNC_SLEEP);
517
518		/* Issue rdcadm -u commands for all remaining sets */
519		for (i = 0; i < sets; i++) {
520			if (sync_done[i])
521				continue;
522
523			/*
524			 * Need to check if autosync was turned off for a set
525			 * while we were sleeping. We could have the case where
526			 * an update sync failed and autosync was disabled
527			 * while we were sleeping and didn't detect the disable.
528			 * See BugID 4814213.
529			 */
530			parms.command = RDC_CMD_STATUS;
531			bcopy((void *)(&rdc_set[i]),
532			    (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
533			if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
534			    ustatus) < 0) {
535				spcs_log("sndr", &ustatus, gettext("%s: "
536				    "status not available for %s:%s, stopping "
537				    "this autosync attempt"), program, tohost,
538				    rdc_set[i].secondary.file);
539				sync_done[i] = 1;
540				syncs_done++;
541				continue;
542			}
543			if (!(parms.rdc_set[0].autosync)) {
544#ifdef DEBUG
545	spcs_log("sndr", NULL, gettext("%s: autosync disabled during sleep, "
546	    "stopping attempt for set %s:%s"), program, tohost,
547	    rdc_set[i].secondary.file);
548#endif
549				sync_done[i] = 1;
550				syncs_done++;
551				continue;
552			}
553
554			(void) sprintf(cmd, "%s -un %s:%s", RDCADM, tohost,
555			    rdc_set[i].secondary.file);
556			spcs_log("sndr", NULL,
557			    gettext("%s: issuing update sync for %s:%s"),
558			    program, tohost, rdc_set[i].secondary.file);
559			(void) system(cmd);
560		}
561
562		/* Issue rdcadm -w commands to wait for updates to finish */
563		for (i = 0; i < sets; i++) {
564			if (sync_done[i])
565				continue;
566
567			(void) sprintf(cmd, "%s -wn %s:%s", RDCADM, tohost,
568			    rdc_set[i].secondary.file);
569			spcs_log("sndr", NULL,
570			    gettext("%s: issuing wait for %s:%s"),
571			    program, tohost, rdc_set[i].secondary.file);
572
573			(void) system(cmd);
574
575			parms.command = RDC_CMD_STATUS;
576			bcopy((void *)(&rdc_set[i]),
577			    (void *)(&parms.rdc_set[0]), sizeof (rdc_set_t));
578
579			if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0,
580			    ustatus) < 0) {
581				spcs_log("sndr", &ustatus,
582				    gettext("%s: status not available for "
583				    "%s:%s, stopping this autosync attempt"),
584				    program, tohost, rdc_set[i].secondary.file);
585				sync_done[i] = 1;
586				syncs_done++;
587				continue;
588			}
589			/* Check if completed OK, failed or autosync off */
590			if (!(parms.rdc_set[0].autosync) ||
591			    !(parms.rdc_set[0].flags & RDC_LOGGING) &&
592			    !(parms.rdc_set[0].flags & RDC_SYNCING)) {
593				sync_done[i] = 1;
594				syncs_done++;
595			}
596		}
597
598		if (syncs_done == sets)
599			break;		/* All completed OK */
600	}
601
602done:
603	if (cfg) {
604		cfg_close(cfg);
605		UNLOCKCFG();
606	}
607	spcs_s_ufree(&ustatus);
608	if (sync_done)
609		free(sync_done);
610	if (rdc_set)
611		free(rdc_set);
612	if (death) { /* bye bye */
613		/*
614		 * if perhaps we lost some race, lets remove this entry from
615		 * the list. Then, if something did go wrong, and we did kill
616		 * a valid thread, it will be detected on the next go around
617		 * of the thread who is looking for new hosts to spawn threads
618		 */
619
620		remove_from_hostlist(tohost);
621		thr_exit(0);
622	}
623
624	(void) sleep(RESYNC_SLEEP);
625}
626
627/*
628 * Wait for notification by the kernel of a sync start or a sync completed OK
629 */
630void *
631wait_sync_event()
632{
633	spcs_s_info_t ustatus;
634	char master[NSC_MAXPATH];
635	char group[NSC_MAXPATH];
636	int state;
637
638	ustatus = spcs_s_ucreate();
639
640	master[0] = '\0';
641	group[0] = '\0';
642
643	/* Never give up */
644	for (;;) {
645		/* Kernel tells us which volume and group the event is for */
646		state = RDC_IOCTL(RDC_SYNC_EVENT, master, group, 0, 0, 0,
647		    ustatus);
648		if (state < SPCS_S_OK) {
649			if (errno != EAGAIN) {
650				spcs_log("sndr", &ustatus,
651				    gettext("%s: update ioctl"),
652				    program);
653				rdc_warn(&ustatus, gettext("update ioctl"));
654				continue;
655			}
656			master[0] = '\0';
657			continue;
658		}
659
660		/*
661		 * If target is mounted at the start of a sync or reverse sync,
662		 * return a negative ack.
663		 */
664		if ((state == RDC_SYNC_START || state == RDC_RSYNC_START) &&
665		    mounted(master)) {
666			spcs_log("sndr", NULL,
667			    gettext("%s: %s has a file system mounted"),
668			    program, master);
669			rdc_warn(NULL,
670			    gettext("%s has a file system mounted"),
671			    master);
672			master[0] = '\0';	/* negative ack */
673			continue;
674		}
675
676		switch (state) {
677		case RDC_SYNC_START:
678			if (group[0])
679				group_start(group);
680			else
681				sync_start(master);
682			break;
683
684		case RDC_SYNC_DONE:
685			if (group[0])
686				group_complete(group);
687			else
688				sync_complete(master);
689			break;
690
691		default:
692			break;
693		}
694	}
695	/* LINTED */
696}
697
698
699/*
700 * A sync has completed OK to a volume not belonging to a group.
701 * Set the state of the ndr_ii config entry to "update".
702 */
703void
704sync_complete(char *master)
705{
706	CFGFILE *cfg = NULL;
707	char buf[CFG_MAX_BUF];
708	char key[CFG_MAX_KEY];
709	int i;
710	int setnumber;
711	int sev;
712
713	LOCKCFG();
714	if ((cfg = cfg_open(NULL)) == NULL) {
715		spcs_log("sndr", NULL,
716		    gettext("%s: error opening config"),
717		    program);
718		rdc_warn(NULL, gettext("error opening config"));
719		UNLOCKCFG();
720		return;
721	}
722	if (!cfg_lock(cfg, CFG_WRLOCK)) {
723		spcs_log("sndr", NULL,
724		    gettext("%s: error locking config"),
725		    program);
726		rdc_warn(NULL, gettext("error locking config"));
727		cfg_close(cfg);
728		UNLOCKCFG();
729		return;
730	}
731
732	/* get ndr_ii entries until a match is found */
733	for (i = 0; ; i++) {
734		setnumber = i + 1;
735
736		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
737		    setnumber);
738		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
739			break;
740		if (strcmp(buf, master) != 0)
741			continue;
742
743		/* Found the matching entry */
744
745		/*
746		 * Set state to "update" so that starting another sync will
747		 * cause a new Point-in-Time Copy snapshot to be taken.
748		 */
749		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
750		    setnumber);
751		if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
752		    (cfg_commit(cfg) < 0)) {
753			spcs_log("sndr", NULL,
754			    gettext("%s: unable to update \"%s\" "
755			    "in configuration storage: %s"),
756			    program, buf, cfg_error(&sev));
757			rdc_warn(NULL,
758			    gettext("unable to update \"%s\" "
759			    "in configuration storage: %s"),
760			    buf, cfg_error(&sev));
761		}
762		break;
763	}
764
765	cfg_close(cfg);
766	UNLOCKCFG();
767}
768
769
770/*
771 * Starting a sync to the specified master volume.
772 * Check the ndr_ii config entries to see if a Point-in-Time Copy
773 * snapshot should be taken.
774 */
775void
776sync_start(char *master)
777{
778	char cmd[256];
779	char buf[CFG_MAX_BUF];
780	char key[CFG_MAX_KEY];
781	CFGFILE *cfg = NULL;
782	int i;
783	int setnumber;
784	int found;
785	int sev;
786	char shadow[NSC_MAXPATH];
787	char bitmap[NSC_MAXPATH];
788	char *ctag = NULL;
789
790	LOCKCFG();
791	if ((cfg = cfg_open(NULL)) == NULL) {
792		spcs_log("sndr", NULL,
793		    gettext("%s: error opening config"),
794		    program);
795		rdc_warn(NULL,
796		    gettext("error opening config"));
797		UNLOCKCFG();
798		return;
799	}
800	if (!cfg_lock(cfg, CFG_RDLOCK)) {
801		spcs_log("sndr", NULL,
802		    gettext("%s: error locking config"),
803		    program);
804		rdc_warn(NULL, gettext("error locking config"));
805		cfg_close(cfg);
806		UNLOCKCFG();
807		return;
808	}
809
810	found = 0;
811	/* get ndr_ii entries until a match is found */
812	for (i = 0; ; i++) {
813		setnumber = i + 1;
814
815		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
816		    setnumber);
817		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
818			break;
819		if (strcmp(buf, master) != 0)
820			continue;
821
822		/* Got a matching entry */
823
824		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
825		    setnumber);
826		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
827			break;
828		(void) strncpy(shadow, buf, NSC_MAXPATH);
829
830		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
831		    setnumber);
832		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
833			break;
834		(void) strncpy(bitmap, buf, NSC_MAXPATH);
835
836		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
837		    setnumber);
838		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
839			break;
840
841		/*
842		 * If an PIT snapshot has already been taken, and syncing did
843		 * not complete, the state will be "noupdate", to indicate we
844		 * should not take another one at this point.
845		 */
846		if (strcmp(buf, NOUPDATE) != 0)
847			found = 1;
848
849		break;
850	}
851
852	if (!found) {
853		cfg_close(cfg);
854		UNLOCKCFG();
855		return;
856	}
857
858	found = 0;
859	/* get ii entries until a match is found */
860	for (i = 0; ; i++) {
861		setnumber = i + 1;
862
863		(void) snprintf(key, sizeof (key), "ii.set%d.shadow",
864		    setnumber);
865		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
866			break;
867		if (strcmp(buf, shadow) != 0)
868			continue;
869
870		/* Matching shadow found, so ii already enabled */
871		found = 1;
872		break;
873	}
874
875	if (found) {
876		/* Already PIT enabled, so just take a snapshot */
877
878		/* Get cluster tag of matching entry */
879		(void) snprintf(key, sizeof (key), "ii.set%d.cnode", setnumber);
880		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
881			if ((strlen(buf) == 0) || (buf[0] == '-'))
882					ctag = "-C local";
883				else
884					ctag = "";
885		(void) sprintf(cmd, "%s %s -u s %s", IIADM, ctag, shadow);
886	} else {
887		/*
888		 * If clustered, need to enable PIT Copy sets in the same
889		 * cluster as the Remote Mirror set
890		 */
891
892		if (clustered) {
893			/* Find a RM set with master as the local volume */
894
895			for (i = 0; i < rdc_maxsets; i++) {
896				setnumber = i + 1;
897				(void) snprintf(key, sizeof (key),
898				    "sndr.set%d.phost", setnumber);
899				if (cfg_get_cstring(cfg, key, buf,
900				    CFG_MAX_BUF) < 0)
901					break;
902
903				if (self_check(buf))
904					(void) snprintf(key, sizeof (key),
905					    "sndr.set%d.primary", setnumber);
906				else
907					(void) snprintf(key, sizeof (key),
908					    "sndr.set%d.secondary", setnumber);
909				if (cfg_get_cstring(cfg, key, buf,
910				    CFG_MAX_BUF) < 0)
911					break;
912
913				if (strcmp(buf, master) != 0)
914					continue;
915
916				/* Get cluster tag of matching entry */
917
918				(void) snprintf(key, sizeof (key),
919				    "sndr.set%d.cnode", setnumber);
920				if (cfg_get_cstring(cfg, key, buf,
921				    CFG_MAX_BUF) < 0)
922					break;
923				if ((strlen(buf) == 0) || (buf[0] == '-'))
924					ctag = strdup("local");
925				else
926					ctag = strdup(buf);
927				break;
928			}
929		}
930
931		/* Not already enabled, so enable a dependent */
932		if (ctag) {
933			(void) sprintf(cmd, "%s -C %s -e dep %s %s %s", IIADM,
934			    ctag, master, shadow, bitmap);
935			free(ctag);
936		} else
937			(void) sprintf(cmd, "%s -e dep %s %s %s", IIADM, master,
938			    shadow, bitmap);
939	}
940
941	cfg_close(cfg);
942
943	if (system(cmd) != 0) {
944		spcs_log("sndr", NULL,
945		    gettext("Point-in-Time Copy snapshot failed for %s %s %s."
946		    " Please check validity of ndr_ii entry"),
947		    master, shadow, bitmap);
948		cfg_close(cfg);
949		UNLOCKCFG();
950		return;
951	}
952
953	/*
954	 * PIT Copy enable or update was fine, so update the ndr_ii entry
955	 * to "noupdate", to prevent invalid point in time copies.
956	 */
957
958	if ((cfg = cfg_open(NULL)) == NULL) {
959		spcs_log("sndr", NULL,
960		    gettext("%s: error opening config"),
961		    program);
962		rdc_warn(NULL,
963		    gettext("error opening config"));
964		UNLOCKCFG();
965		return;
966	}
967	if (!cfg_lock(cfg, CFG_WRLOCK)) {
968		spcs_log("sndr", NULL,
969		    gettext("%s: error locking config"),
970		    program);
971		rdc_warn(NULL, gettext("error locking config"));
972		cfg_close(cfg);
973		UNLOCKCFG();
974		return;
975	}
976
977	/* get ndr_ii entries until a match is found */
978	for (i = 0; ; i++) {
979		setnumber = i + 1;
980
981		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
982		    setnumber);
983		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
984			break;
985		if (strcmp(buf, shadow) != 0)
986			continue;
987
988		/* Found the matching entry */
989
990		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
991		    setnumber);
992		if ((cfg_put_cstring(cfg, key, NOUPDATE,
993			strlen(NOUPDATE)) < 0) || (cfg_commit(cfg) < 0)) {
994			spcs_log("sndr", NULL,
995			    gettext("%s: unable to update \"%s\" "
996			    "in configuration storage: %s"),
997			    program, buf, cfg_error(&sev));
998			rdc_warn(NULL,
999			    gettext("unable to update \"%s\" "
1000			    "in configuration storage: %s"),
1001			    buf, cfg_error(&sev));
1002		}
1003		break;
1004	}
1005	cfg_close(cfg);
1006	UNLOCKCFG();
1007}
1008
1009void
1010cleanup_hostlist(rdc_status_t *rdc_info)
1011{
1012	int i, j, k;
1013	char *host, *exhost;
1014
1015
1016	(void) mutex_lock(&host_list->hosts_mutex);
1017	for (i = 0; i < host_list->numhosts; i++) {
1018		int found = 0;
1019		for (j = 0; (j < rdc_maxsets) && !found; j++) {
1020			if (!rdc_info->rdc_set[j].flags & RDC_ENABLED)
1021				continue;
1022			if ((!host_list->configured[i]) ||
1023			    (host_list->hosts[i] == '\0')) {
1024				(void) mutex_unlock(&host_list->hosts_mutex);
1025				return;
1026			}
1027
1028			host = rdc_info->rdc_set[j].secondary.intf;
1029			if (strcmp(host_list->hosts[i], host) == 0)
1030				found++;
1031		}
1032		if (j == rdc_maxsets) {
1033			/*
1034			 * this set is not in the kernel, so remove from list
1035			 */
1036			exhost = host_list->hosts[i];
1037			if (exhost) {
1038				free(exhost);
1039				exhost = NULL;
1040			}
1041
1042			k = i;
1043			while (k < host_list->numhosts) {
1044			    host_list->hosts[k] = k < host_list->numhosts - 1 ?
1045				host_list->hosts[k+1] : NULL;
1046			    k++;
1047			}
1048			host_list->numhosts--;
1049
1050			bcopy(&host_list->configured[i+1],
1051			    &host_list->configured[i],
1052			    (MAXHOSTS - i + 1) * sizeof (int));
1053			host_list->configured[MAXHOSTS - 1] = 0;
1054		}
1055	}
1056	(void) mutex_unlock(&host_list->hosts_mutex);
1057}
1058
1059/*
1060 * explicity remove a host from the host list
1061 * also update the configured array
1062 * called in rdc_sync, just before exiting a thread.
1063 */
1064void
1065remove_from_hostlist(char *host)
1066{
1067	int i, k;
1068	char *exhost;
1069
1070	/* why bother? */
1071	if ((!host) || (host[0] == '\0'))
1072		return;
1073
1074	(void) mutex_lock(&host_list->hosts_mutex);
1075	for (i = 0; i < host_list->numhosts; i++) {
1076		if (strcmp(host, host_list->hosts[i]) == 0) { /* found it */
1077			exhost = host_list->hosts[i];
1078			if (exhost) {
1079				free(exhost);
1080				exhost = NULL;
1081			}
1082			k = i;
1083			while (k < host_list->numhosts) {
1084			    host_list->hosts[k] = k < host_list->numhosts - 1 ?
1085				host_list->hosts[k+1] : NULL;
1086			    k++;
1087			}
1088			host_list->numhosts--;
1089			bcopy(&host_list->configured[i+1],
1090			    &host_list->configured[i],
1091			    (MAXHOSTS - i + 1) * sizeof (int));
1092			host_list->configured[MAXHOSTS - 1] = 0;
1093		}
1094
1095	}
1096	(void) mutex_unlock(&host_list->hosts_mutex);
1097}
1098/*
1099 * Check to see if this host isn't in our list, so needs a new rdcsyncd proc
1100 */
1101int
1102isnewhost(char *host)
1103{
1104	int i;
1105	int new;
1106
1107	if (self_check(host)) {
1108		return (0);
1109	}
1110
1111	(void) mutex_lock(&host_list->hosts_mutex);
1112	new = 1;
1113	for (i = 0; i < MAXHOSTS; i++) {
1114		if (host_list->configured[i] == 0) {
1115			host_list->configured[i] = 1;
1116			host_list->hosts[i] = strdup(host);
1117			host_list->numhosts++;
1118			break;
1119		}
1120		if (strcmp(host, host_list->hosts[i]) == 0) {
1121			new = 0;
1122			break;
1123		}
1124	}
1125	(void) mutex_unlock(&host_list->hosts_mutex);
1126	if (i == MAXHOSTS)
1127		new = 0;
1128	return (new);
1129}
1130
1131
1132/*
1133 * Look for a matching volume name in our remembered list.
1134 */
1135int
1136volume_match(char *buf, char **volume_list, int volumes)
1137{
1138	int i;
1139	char *vol;
1140
1141	for (i = 0; i < volumes; i++) {
1142		vol = volume_list[i];
1143		if (strcmp(buf, vol) == 0) {
1144			return (1);
1145		}
1146	}
1147	return (0);
1148}
1149
1150
1151/*
1152 * A sync has completed to a group. We can only update the ndr_ii entries
1153 * if all the members of the group have completed their syncs OK.
1154 * It would be bad to allow some members of the group to have PIT Copy snapshots
1155 * taken and others not, as they need to be consistent.
1156 */
1157void
1158group_complete(char *group)
1159{
1160	char **volumes = NULL;
1161	spcs_s_info_t ustatus;
1162	rdc_config_t parms = { 0 };
1163	char buf[CFG_MAX_BUF];
1164	char key[CFG_MAX_KEY];
1165	CFGFILE *cfg = NULL;
1166	int i;
1167	int setnumber;
1168	int found;
1169	int replicating = 0;
1170	char primary[NSC_MAXPATH];
1171	char secondary[NSC_MAXPATH];
1172	char phost[MAX_RDC_HOST_SIZE];
1173	char shost[MAX_RDC_HOST_SIZE];
1174	rdc_set_t *rdc_set;
1175	int sev;
1176	char *local_file;
1177	int size;
1178
1179	ustatus = spcs_s_ucreate();
1180
1181	size = sizeof (char *) * rdc_maxsets;
1182	volumes = malloc(size);
1183	if (volumes == NULL) {
1184		spcs_log("sndr", NULL,
1185		    gettext("%s: unable to allocate %ld bytes"),
1186		    program, size);
1187		rdc_warn(NULL,
1188			gettext("unable to allocate %ld bytes"), size);
1189		goto done;
1190	}
1191	bzero(volumes, size);
1192
1193	/*
1194	 * If all members of this group are replicating
1195	 * set ii_ndr state to "update". Otherwise leave them alone.
1196	 */
1197	LOCKCFG();
1198	if ((cfg = cfg_open(NULL)) == NULL) {
1199		spcs_log("sndr", NULL,
1200		    gettext("%s: error opening lconfig"),
1201		    program);
1202		rdc_warn(NULL, gettext("error opening config"));
1203		UNLOCKCFG();
1204		goto done;
1205	}
1206
1207	if (!cfg_lock(cfg, CFG_RDLOCK)) {
1208		spcs_log("sndr", NULL,
1209		    gettext("%s: error locking config"),
1210		    program);
1211		rdc_warn(NULL, gettext("error locking config"));
1212		goto done;
1213	}
1214
1215	found = 0;
1216
1217	/* get all RM entries, with a matching group, that are replicating */
1218	for (i = 0; i < rdc_maxsets; i++) {
1219		setnumber = i + 1;
1220
1221		(void) snprintf(key, sizeof (key),
1222		    "sndr.set%d.group", setnumber);
1223		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1224			break;
1225
1226		if (strcmp(buf, group) != 0)
1227			continue;
1228
1229		/* Found a matching entry */
1230
1231		(void) snprintf(key, sizeof (key),
1232		    "sndr.set%d.primary", setnumber);
1233		if (cfg_get_cstring(cfg, key, primary, sizeof (primary)) < 0)
1234			break;
1235		(void) strcpy(parms.rdc_set->primary.file, primary);
1236
1237		(void) snprintf(key, sizeof (key),
1238		    "sndr.set%d.phost", setnumber);
1239		if (cfg_get_cstring(cfg, key, phost, sizeof (phost)) < 0)
1240			break;
1241		(void) strcpy(parms.rdc_set->primary.intf, phost);
1242
1243		(void) snprintf(key, sizeof (key),
1244		    "sndr.set%d.secondary", setnumber);
1245		if (cfg_get_cstring(cfg, key, secondary,
1246				sizeof (secondary)) < 0)
1247			break;
1248		(void) strcpy(parms.rdc_set->secondary.file, secondary);
1249
1250		(void) snprintf(key, sizeof (key),
1251		    "sndr.set%d.shost", setnumber);
1252		if (cfg_get_cstring(cfg, key, shost, sizeof (shost)) < 0)
1253			break;
1254		(void) strcpy(parms.rdc_set->secondary.intf, shost);
1255
1256		parms.command = RDC_CMD_STATUS;
1257		if (RDC_IOCTL(RDC_CONFIG, &parms, NULL, 0, 0, 0, ustatus) < 0) {
1258			continue;
1259		}
1260
1261		/* We found a matching set */
1262		found++;
1263
1264		if (self_check(phost))
1265			local_file = primary;
1266		else
1267			local_file = secondary;
1268
1269		rdc_set = &parms.rdc_set[0];
1270		if (!(rdc_set->flags & RDC_LOGGING) &&
1271		    !(rdc_set->flags & RDC_SYNCING)) {
1272			volumes[replicating] = strdup(local_file);
1273			if (volumes[replicating] == NULL) {
1274				size = strlen(local_file);
1275				spcs_log("sndr", NULL,
1276				    gettext("%s: unable to allocate %ld bytes"),
1277				    program, size);
1278				rdc_warn(NULL,
1279				    gettext("unable to allocate %ld bytes"),
1280				    size);
1281				goto done;
1282			}
1283			/* We remember all replicating sets */
1284			replicating++;
1285		} else
1286			break;		/* Not all replicating, so done */
1287	}
1288
1289	if (found != replicating)
1290		goto done;
1291
1292	/* All replicating, so update ndr_ii state fields */
1293
1294	cfg_unlock(cfg);
1295
1296	if (!cfg_lock(cfg, CFG_WRLOCK)) {
1297		spcs_log("sndr", NULL,
1298		    gettext("%s: error locking lconfig"),
1299		    program);
1300		rdc_warn(NULL, gettext("error locking config"));
1301		goto done;
1302	}
1303
1304	/*
1305	 * Search through the ndr_ii entries for entries
1306	 * that match the saved secondary volume names.
1307	 * Set state to "update".
1308	 */
1309
1310	for (i = 0; ; i++) {
1311		setnumber = i + 1;
1312
1313		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1314		    setnumber);
1315		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1316			break;
1317
1318		if (!volume_match(buf, volumes, found)) {
1319			continue;
1320		}
1321
1322		/* Got a matching entry */
1323
1324		(void) snprintf(key, sizeof (key),
1325		    "ndr_ii.set%d.state", setnumber);
1326		if ((cfg_put_cstring(cfg, key, UPDATE, strlen(UPDATE)) < 0) ||
1327		    (cfg_commit(cfg) < 0)) {
1328			spcs_log("sndr", NULL,
1329			    gettext("%s: unable to update \"%s\" "
1330			    "in configuration storage: %s"),
1331			    program, buf, cfg_error(&sev));
1332			rdc_warn(NULL,
1333			    gettext("unable to update \"%s\" "
1334			    "in configuration storage: %s"),
1335			    buf, cfg_error(&sev));
1336		}
1337	}
1338
1339
1340done:
1341	if (cfg) {
1342		cfg_close(cfg);
1343		UNLOCKCFG();
1344	}
1345	spcs_s_ufree(&ustatus);
1346	if (volumes) {
1347		for (i = 0; i < replicating; i++)
1348			free(volumes[i]);
1349		free(volumes);
1350	}
1351}
1352
1353
1354/*
1355 * Sync started to a member of a group.
1356 * If all members of the group are in ndr_ii state "update" then take an PIT
1357 * snapshot on all of them. This will provide a consistent point-in-time
1358 * copy until whatever syncs take place are all completed.
1359 */
1360void
1361group_start(char *group)
1362{
1363	char **masters = NULL;
1364	char **shadows = NULL;
1365	char **bitmaps = NULL;
1366	char cmd[256];
1367	char buf[CFG_MAX_BUF];
1368	char key[CFG_MAX_KEY];
1369	CFGFILE *cfg = NULL;
1370	int i;
1371	int j;
1372	int setnumber;
1373	int found;
1374	int sndr_sets = 0;
1375	int update_needed = 0;
1376	int sev;
1377	char *ctag = NULL;
1378	int commit = 0;
1379	int size;
1380
1381	size = sizeof (char *) * rdc_maxsets;
1382	masters = malloc(size);
1383	if (masters == NULL) {
1384		spcs_log("sndr", NULL,
1385		    gettext("%s: unable to allocate %ld bytes"),
1386		    program, size);
1387		rdc_warn(NULL,
1388			gettext("unable to allocate %ld bytes"), size);
1389		goto done;
1390	}
1391	bzero(masters, size);
1392	shadows = malloc(size);
1393	if (shadows == NULL) {
1394		spcs_log("sndr", NULL,
1395		    gettext("%s: unable to allocate %ld bytes"),
1396		    program, size);
1397		rdc_warn(NULL,
1398			gettext("unable to allocate %ld bytes"), size);
1399		goto done;
1400	}
1401	bzero(shadows, size);
1402	bitmaps = malloc(size);
1403	if (bitmaps == NULL) {
1404		spcs_log("sndr", NULL,
1405		    gettext("%s: unable to allocate %ld bytes"),
1406		    program, size);
1407		rdc_warn(NULL,
1408			gettext("unable to allocate %ld bytes"), size);
1409		goto done;
1410	}
1411	bzero(bitmaps, size);
1412
1413	LOCKCFG();
1414	if ((cfg = cfg_open(NULL)) == NULL) {
1415		spcs_log("sndr", NULL,
1416		    gettext("%s: error opening config"),
1417		    program);
1418		rdc_warn(NULL,
1419		    gettext("error opening config"));
1420		UNLOCKCFG();
1421		goto done;
1422	}
1423
1424	if (!cfg_lock(cfg, CFG_WRLOCK)) {
1425		spcs_log("sndr", NULL,
1426		    gettext("%s: error locking config"),
1427		    program);
1428		rdc_warn(NULL, gettext("error locking config"));
1429		goto done;
1430	}
1431
1432	/* Now get all Remote Mirror entries with a matching group */
1433	for (i = 0; i < rdc_maxsets; i++) {
1434		setnumber = i + 1;
1435
1436		(void) snprintf(key, sizeof (key),
1437		    "sndr.set%d.group", setnumber);
1438		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1439			break;
1440
1441		if (strcmp(buf, group) != 0)
1442			continue;
1443
1444		/* Found a matching entry */
1445
1446		(void) snprintf(key, sizeof (key),
1447		    "sndr.set%d.phost", setnumber);
1448		if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1449			break;
1450
1451		if (self_check(buf)) {
1452			(void) snprintf(key, sizeof (key), "sndr.set%d.primary",
1453			    setnumber);
1454		} else {
1455			(void) snprintf(key, sizeof (key),
1456			    "sndr.set%d.secondary", setnumber);
1457		}
1458		if (cfg_get_cstring(cfg, key, buf, sizeof (buf)) < 0)
1459			break;
1460
1461		masters[sndr_sets] = strdup(buf);
1462		if (masters[sndr_sets] == NULL) {
1463			size = strlen(buf);
1464			spcs_log("sndr", NULL,
1465			    gettext("%s: unable to allocate %ld bytes"),
1466			    program, size);
1467			rdc_warn(NULL,
1468				gettext("unable to allocate %ld bytes"), size);
1469			goto done;
1470		}
1471		sndr_sets++;
1472
1473		if (ctag == NULL && clustered) {
1474			/* Get cluster tag of matching entry */
1475
1476			(void) snprintf(key, sizeof (key), "sndr.set%d.cnode",
1477			    setnumber);
1478			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) >= 0)
1479				ctag = strdup(buf);
1480		}
1481	}
1482
1483	/*
1484	 * Search through the ndr_ii entries for entries
1485	 * that match the saved local volume names and are in "update" state.
1486	 */
1487
1488	update_needed = 0;
1489
1490	for (i = 0; ; i++) {
1491		setnumber = i + 1;
1492
1493		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.secondary",
1494		    setnumber);
1495		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1496			break;
1497
1498		if (!volume_match(buf, masters, sndr_sets))
1499			continue;
1500
1501		/* Got a matching entry */
1502
1503		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.shadow",
1504		    setnumber);
1505		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1506			break;
1507		shadows[update_needed] = strdup(buf);
1508		if (shadows[update_needed] == NULL) {
1509			size = strlen(buf);
1510			spcs_log("sndr", NULL,
1511			    gettext("%s: unable to allocate %ld bytes"),
1512			    program, size);
1513			rdc_warn(NULL,
1514				gettext("unable to allocate %ld bytes"), size);
1515			goto done;
1516		}
1517
1518		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.bitmap",
1519		    setnumber);
1520		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1521			break;
1522		}
1523		bitmaps[update_needed] = strdup(buf);
1524		if (bitmaps[update_needed] == NULL) {
1525			size = strlen(buf);
1526			spcs_log("sndr", NULL,
1527			    gettext("%s: unable to allocate %ld bytes"),
1528			    program, size);
1529			rdc_warn(NULL,
1530				gettext("unable to allocate %ld bytes"), size);
1531			goto done;
1532		}
1533
1534		(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1535		    setnumber);
1536		if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0) {
1537			break;
1538		}
1539		if (strcmp(buf, UPDATE) != 0) {
1540			break;
1541		}
1542
1543		update_needed++;
1544	}
1545
1546	if (update_needed != sndr_sets) {
1547#ifdef DEBUG
1548		spcs_log("sndr", NULL,
1549		    gettext("%s: group sync: no Point-in-Time Copy snapshot "
1550			    "for %s"), program, group);
1551#endif
1552		goto done;
1553	}
1554
1555	/* All RM sets in the group have an ndr_ii entry in "update" state */
1556
1557	/* Issue PIT Copy snapshot commands for all sets in the group */
1558	for (j = 0; j < sndr_sets; j++) {
1559		found = 0;
1560
1561		/* get ii entries until a match is found */
1562		for (i = 0; ; i++) {
1563			setnumber = i + 1;
1564
1565			(void) snprintf(key, sizeof (key), "ii.set%d.shadow",
1566			    setnumber);
1567			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1568				break;
1569			if (strcmp(buf, shadows[j]) != 0)
1570				continue;
1571
1572			/* Matching shadow found, so ii already enabled */
1573			found = 1;
1574			break;
1575		}
1576
1577		if (commit)
1578			if (cfg_commit(cfg) < 0)
1579				rdc_warn(NULL, gettext("commit config error"));
1580		cfg_close(cfg);
1581
1582		if (found) {
1583			(void) sprintf(cmd, "%s -u s %s", IIADM, shadows[j]);
1584		} else {
1585			if (ctag) {
1586				(void) sprintf(cmd, "%s -C %s -e dep %s %s %s",
1587				    IIADM, ctag, masters[j], shadows[j],
1588				    bitmaps[j]);
1589				free(ctag);
1590				ctag = NULL;
1591			} else
1592				(void) sprintf(cmd, "%s -e dep %s %s %s", IIADM,
1593				    masters[j], shadows[j], bitmaps[j]);
1594		}
1595
1596		if (system(cmd) != 0) {
1597			spcs_log("sndr", NULL,
1598			    gettext("%s: group sync: Point-in-Time Copy"
1599			    " snapshot failed for %s"),
1600			    program, masters[j]);
1601
1602			goto done;
1603		}
1604
1605		if ((cfg = cfg_open(NULL)) == NULL) {
1606			spcs_log("sndr", NULL,
1607			    gettext("%s: error opening config"),
1608			    program);
1609			rdc_warn(NULL,
1610			    gettext("error opening config"));
1611			goto done;
1612		}
1613		if (!cfg_lock(cfg, CFG_WRLOCK)) {
1614			spcs_log("sndr", NULL,
1615			    gettext("%s: error locking config"),
1616			    program);
1617			rdc_warn(NULL, gettext("error locking config"));
1618			goto done;
1619		}
1620		commit = 0;
1621
1622		/* PIT enable or update was fine, so update the ndr_ii entry */
1623
1624		/* get ndr_ii entries until a match is found */
1625		for (i = 0; ; i++) {
1626			setnumber = i + 1;
1627
1628			(void) snprintf(key, sizeof (key),
1629			    "ndr_ii.set%d.shadow", setnumber);
1630			if (cfg_get_cstring(cfg, key, buf, CFG_MAX_BUF) < 0)
1631				break;
1632			if (strcmp(buf, shadows[j]) != 0)
1633				continue;
1634
1635			/* Found the matching entry */
1636
1637			(void) snprintf(key, sizeof (key), "ndr_ii.set%d.state",
1638			    setnumber);
1639			if (cfg_put_cstring(cfg, key, NOUPDATE,
1640			    strlen(NOUPDATE)) < 0) {
1641				spcs_log("sndr", NULL,
1642				    gettext("%s: unable to update \"%s\" "
1643				    "in configuration storage: %s"),
1644				    program, buf, cfg_error(&sev));
1645				rdc_warn(NULL,
1646				    gettext("unable to update \"%s\" "
1647				    "in configuration storage: %s"),
1648				    buf, cfg_error(&sev));
1649			} else
1650				commit = 1;
1651			break;
1652		}
1653	}
1654
1655	if (commit)
1656		if (cfg_commit(cfg) < 0)
1657			rdc_warn(NULL, gettext("commit config error"));
1658
1659	spcs_log("sndr", NULL,
1660	    gettext("%s: group sync: Point-in-Time Copy snapshots completed "
1661		    "for %s"), program, group);
1662
1663done:
1664	if (ctag)
1665		free(ctag);
1666
1667	if (cfg) {
1668		cfg_close(cfg);
1669		UNLOCKCFG();
1670	}
1671
1672	if (masters) {
1673		for (i = 0; i < sndr_sets; i++) {
1674			if (masters[i])
1675				free(masters[i]);
1676		}
1677		free(masters);
1678	}
1679
1680	if (shadows) {
1681		for (i = 0; i < update_needed; i++) {
1682			if (shadows[i])
1683				free(shadows[i]);
1684		}
1685		free(shadows);
1686	}
1687
1688	if (bitmaps) {
1689		for (i = 0; i < update_needed; i++) {
1690			if (bitmaps[i])
1691				free(bitmaps[i]);
1692		}
1693		free(bitmaps);
1694	}
1695}
1696