Deleted Added
sdiff udiff text old ( 211981 ) new ( 211982 )
full compact
primary.c (211981) primary.c (211982)
1/*-
2 * Copyright (c) 2009 The FreeBSD Foundation
3 * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
4 * All rights reserved.
5 *
6 * This software was developed by Pawel Jakub Dawidek under sponsorship from
7 * the FreeBSD Foundation.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2009 The FreeBSD Foundation
3 * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
4 * All rights reserved.
5 *
6 * This software was developed by Pawel Jakub Dawidek under sponsorship from
7 * the FreeBSD Foundation.
8 *

--- 15 unchanged lines hidden (view full) ---

24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD: head/sbin/hastd/primary.c 211981 2010-08-29 22:55:21Z pjd $");
32__FBSDID("$FreeBSD: head/sbin/hastd/primary.c 211982 2010-08-30 00:06:05Z pjd $");
33
34#include <sys/types.h>
35#include <sys/time.h>
36#include <sys/bio.h>
37#include <sys/disk.h>
38#include <sys/refcount.h>
39#include <sys/stat.h>
40
41#include <geom/gate/g_gate.h>
42
43#include <assert.h>
44#include <err.h>
45#include <errno.h>
46#include <fcntl.h>
47#include <libgeom.h>
48#include <pthread.h>
33
34#include <sys/types.h>
35#include <sys/time.h>
36#include <sys/bio.h>
37#include <sys/disk.h>
38#include <sys/refcount.h>
39#include <sys/stat.h>
40
41#include <geom/gate/g_gate.h>
42
43#include <assert.h>
44#include <err.h>
45#include <errno.h>
46#include <fcntl.h>
47#include <libgeom.h>
48#include <pthread.h>
49#include <signal.h>
49#include <stdint.h>
50#include <stdio.h>
51#include <string.h>
52#include <sysexits.h>
53#include <unistd.h>
54
55#include <activemap.h>
56#include <nv.h>

--- 70 unchanged lines hidden (view full) ---

127 */
128static bool sync_inprogress;
129static pthread_mutex_t sync_lock;
130static pthread_cond_t sync_cond;
131/*
132 * The lock below allows to synchornize access to remote connections.
133 */
134static pthread_rwlock_t *hio_remote_lock;
50#include <stdint.h>
51#include <stdio.h>
52#include <string.h>
53#include <sysexits.h>
54#include <unistd.h>
55
56#include <activemap.h>
57#include <nv.h>

--- 70 unchanged lines hidden (view full) ---

128 */
129static bool sync_inprogress;
130static pthread_mutex_t sync_lock;
131static pthread_cond_t sync_cond;
132/*
133 * The lock below allows to synchornize access to remote connections.
134 */
135static pthread_rwlock_t *hio_remote_lock;
135static pthread_mutex_t hio_guard_lock;
136static pthread_cond_t hio_guard_cond;
137
138/*
139 * Lock to synchronize metadata updates. Also synchronize access to
140 * hr_primary_localcnt and hr_primary_remotecnt fields.
141 */
142static pthread_mutex_t metadata_lock;
143
144/*
145 * Maximum number of outstanding I/O requests.
146 */
147#define HAST_HIO_MAX 256
148/*
149 * Number of components. At this point there are only two components: local
150 * and remote, but in the future it might be possible to use multiple local
151 * and remote components.
152 */
153#define HAST_NCOMPONENTS 2
154/*
136
137/*
138 * Lock to synchronize metadata updates. Also synchronize access to
139 * hr_primary_localcnt and hr_primary_remotecnt fields.
140 */
141static pthread_mutex_t metadata_lock;
142
143/*
144 * Maximum number of outstanding I/O requests.
145 */
146#define HAST_HIO_MAX 256
147/*
148 * Number of components. At this point there are only two components: local
149 * and remote, but in the future it might be possible to use multiple local
150 * and remote components.
151 */
152#define HAST_NCOMPONENTS 2
153/*
155 * Number of seconds to sleep between keepalive packets.
154 * Number of seconds to sleep between reconnect retries or keepalive packets.
156 */
155 */
157#define KEEPALIVE_SLEEP 10
158/*
159 * Number of seconds to sleep between reconnect retries.
160 */
161#define RECONNECT_SLEEP 5
156#define RETRY_SLEEP 10
162
163#define ISCONNECTED(res, no) \
164 ((res)->hr_remotein != NULL && (res)->hr_remoteout != NULL)
165
166#define QUEUE_INSERT1(hio, name, ncomp) do { \
167 bool _wakeup; \
168 \
169 mtx_lock(&hio_##name##_list_lock[(ncomp)]); \

--- 55 unchanged lines hidden (view full) ---

225static void *ggate_recv_thread(void *arg);
226static void *local_send_thread(void *arg);
227static void *remote_send_thread(void *arg);
228static void *remote_recv_thread(void *arg);
229static void *ggate_send_thread(void *arg);
230static void *sync_thread(void *arg);
231static void *guard_thread(void *arg);
232
157
158#define ISCONNECTED(res, no) \
159 ((res)->hr_remotein != NULL && (res)->hr_remoteout != NULL)
160
161#define QUEUE_INSERT1(hio, name, ncomp) do { \
162 bool _wakeup; \
163 \
164 mtx_lock(&hio_##name##_list_lock[(ncomp)]); \

--- 55 unchanged lines hidden (view full) ---

220static void *ggate_recv_thread(void *arg);
221static void *local_send_thread(void *arg);
222static void *remote_send_thread(void *arg);
223static void *remote_recv_thread(void *arg);
224static void *ggate_send_thread(void *arg);
225static void *sync_thread(void *arg);
226static void *guard_thread(void *arg);
227
233static void sighandler(int sig);
228static void
229dummy_sighandler(int sig __unused)
230{
231 /* Nothing to do. */
232}
234
235static void
236cleanup(struct hast_resource *res)
237{
238 int rerrno;
239
240 /* Remember errno. */
241 rerrno = errno;

--- 72 unchanged lines hidden (view full) ---

314 return (strcmp(res->hr_remoteaddr, "none") != 0);
315}
316
317static void
318init_environment(struct hast_resource *res __unused)
319{
320 struct hio *hio;
321 unsigned int ii, ncomps;
233
234static void
235cleanup(struct hast_resource *res)
236{
237 int rerrno;
238
239 /* Remember errno. */
240 rerrno = errno;

--- 72 unchanged lines hidden (view full) ---

313 return (strcmp(res->hr_remoteaddr, "none") != 0);
314}
315
316static void
317init_environment(struct hast_resource *res __unused)
318{
319 struct hio *hio;
320 unsigned int ii, ncomps;
321 sigset_t mask;
322
323 /*
324 * In the future it might be per-resource value.
325 */
326 ncomps = HAST_NCOMPONENTS;
327
328 /*
329 * Allocate memory needed by lists.

--- 54 unchanged lines hidden (view full) ---

384 TAILQ_INIT(&hio_recv_list[ii]);
385 mtx_init(&hio_recv_list_lock[ii]);
386 cv_init(&hio_recv_list_cond[ii]);
387 rw_init(&hio_remote_lock[ii]);
388 }
389 TAILQ_INIT(&hio_done_list);
390 mtx_init(&hio_done_list_lock);
391 cv_init(&hio_done_list_cond);
322
323 /*
324 * In the future it might be per-resource value.
325 */
326 ncomps = HAST_NCOMPONENTS;
327
328 /*
329 * Allocate memory needed by lists.

--- 54 unchanged lines hidden (view full) ---

384 TAILQ_INIT(&hio_recv_list[ii]);
385 mtx_init(&hio_recv_list_lock[ii]);
386 cv_init(&hio_recv_list_cond[ii]);
387 rw_init(&hio_remote_lock[ii]);
388 }
389 TAILQ_INIT(&hio_done_list);
390 mtx_init(&hio_done_list_lock);
391 cv_init(&hio_done_list_cond);
392 mtx_init(&hio_guard_lock);
393 cv_init(&hio_guard_cond);
394 mtx_init(&metadata_lock);
395
396 /*
397 * Allocate requests pool and initialize requests.
398 */
399 for (ii = 0; ii < HAST_HIO_MAX; ii++) {
400 hio = malloc(sizeof(*hio));
401 if (hio == NULL) {

--- 24 unchanged lines hidden (view full) ---

426 hio->hio_ggio.gctl_length = MAXPHYS;
427 hio->hio_ggio.gctl_error = 0;
428 TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_free_next);
429 }
430
431 /*
432 * Turn on signals handling.
433 */
392 mtx_init(&metadata_lock);
393
394 /*
395 * Allocate requests pool and initialize requests.
396 */
397 for (ii = 0; ii < HAST_HIO_MAX; ii++) {
398 hio = malloc(sizeof(*hio));
399 if (hio == NULL) {

--- 24 unchanged lines hidden (view full) ---

424 hio->hio_ggio.gctl_length = MAXPHYS;
425 hio->hio_ggio.gctl_error = 0;
426 TAILQ_INSERT_HEAD(&hio_free_list, hio, hio_free_next);
427 }
428
429 /*
430 * Turn on signals handling.
431 */
434 signal(SIGINT, sighandler);
435 signal(SIGTERM, sighandler);
436 signal(SIGHUP, sighandler);
437 signal(SIGCHLD, sighandler);
432 /* Because SIGCHLD is ignored by default, setup dummy handler for it. */
433 PJDLOG_VERIFY(signal(SIGCHLD, dummy_sighandler) != SIG_ERR);
434 PJDLOG_VERIFY(sigfillset(&mask) == 0);
435 PJDLOG_VERIFY(sigprocmask(SIG_SETMASK, &mask, NULL) == 0);
438}
439
440static void
441init_local(struct hast_resource *res)
442{
443 unsigned char *buf;
444 size_t mapsize;
445

--- 442 unchanged lines hidden (view full) ---

888 rw_unlock(&hio_remote_lock[ncomp]);
889
890 pjdlog_warning("Disconnected from %s.", res->hr_remoteaddr);
891
892 /*
893 * Stop synchronization if in-progress.
894 */
895 sync_stop();
436}
437
438static void
439init_local(struct hast_resource *res)
440{
441 unsigned char *buf;
442 size_t mapsize;
443

--- 442 unchanged lines hidden (view full) ---

886 rw_unlock(&hio_remote_lock[ncomp]);
887
888 pjdlog_warning("Disconnected from %s.", res->hr_remoteaddr);
889
890 /*
891 * Stop synchronization if in-progress.
892 */
893 sync_stop();
896
897 /*
898 * Wake up guard thread (if we are not called from within guard thread),
899 * so it can immediately start reconnect.
900 */
901 if (!mtx_owned(&hio_guard_lock)) {
902 mtx_lock(&hio_guard_lock);
903 cv_signal(&hio_guard_cond);
904 mtx_unlock(&hio_guard_lock);
905 }
906}
907
908/*
909 * Thread receives ggate I/O requests from the kernel and passes them to
910 * appropriate threads:
911 * WRITE - always goes to both local_send and remote_send threads
912 * READ (when the block is up-to-date on local component) -
913 * only local_send thread

--- 815 unchanged lines hidden (view full) ---

1729 hio);
1730 QUEUE_INSERT2(hio, free);
1731 }
1732 /* NOTREACHED */
1733 return (NULL);
1734}
1735
1736static void
894}
895
896/*
897 * Thread receives ggate I/O requests from the kernel and passes them to
898 * appropriate threads:
899 * WRITE - always goes to both local_send and remote_send threads
900 * READ (when the block is up-to-date on local component) -
901 * only local_send thread

--- 815 unchanged lines hidden (view full) ---

1717 hio);
1718 QUEUE_INSERT2(hio, free);
1719 }
1720 /* NOTREACHED */
1721 return (NULL);
1722}
1723
1724static void
1737sighandler(int sig)
1738{
1739 bool unlock;
1740
1741 switch (sig) {
1742 case SIGINT:
1743 case SIGTERM:
1744 sigexit_received = true;
1745 break;
1746 case SIGHUP:
1747 sighup_received = true;
1748 break;
1749 case SIGCHLD:
1750 sigchld_received = true;
1751 break;
1752 default:
1753 assert(!"invalid condition");
1754 }
1755 /*
1756 * Racy, but if we cannot obtain hio_guard_lock here, we don't
1757 * want to risk deadlock.
1758 */
1759 unlock = mtx_trylock(&hio_guard_lock);
1760 cv_signal(&hio_guard_cond);
1761 if (unlock)
1762 mtx_unlock(&hio_guard_lock);
1763}
1764
1765static void
1766config_reload(void)
1767{
1768 struct hastd_config *newcfg;
1769 struct hast_resource *res;
1770 unsigned int ii, ncomps;
1771 int modified;
1772
1773 pjdlog_info("Reloading configuration...");

--- 194 unchanged lines hidden (view full) ---

1968 * Thread guards remote connections and reconnects when needed, handles
1969 * signals, etc.
1970 */
1971static void *
1972guard_thread(void *arg)
1973{
1974 struct hast_resource *res = arg;
1975 unsigned int ii, ncomps;
1725config_reload(void)
1726{
1727 struct hastd_config *newcfg;
1728 struct hast_resource *res;
1729 unsigned int ii, ncomps;
1730 int modified;
1731
1732 pjdlog_info("Reloading configuration...");

--- 194 unchanged lines hidden (view full) ---

1927 * Thread guards remote connections and reconnects when needed, handles
1928 * signals, etc.
1929 */
1930static void *
1931guard_thread(void *arg)
1932{
1933 struct hast_resource *res = arg;
1934 unsigned int ii, ncomps;
1935 struct timespec timeout;
1976 time_t lastcheck, now;
1936 time_t lastcheck, now;
1977 int timeout;
1937 sigset_t mask;
1938 int signo;
1978
1979 ncomps = HAST_NCOMPONENTS;
1980 lastcheck = time(NULL);
1981
1939
1940 ncomps = HAST_NCOMPONENTS;
1941 lastcheck = time(NULL);
1942
1943 PJDLOG_VERIFY(sigemptyset(&mask) == 0);
1944 PJDLOG_VERIFY(sigaddset(&mask, SIGHUP) == 0);
1945 PJDLOG_VERIFY(sigaddset(&mask, SIGINT) == 0);
1946 PJDLOG_VERIFY(sigaddset(&mask, SIGTERM) == 0);
1947 PJDLOG_VERIFY(sigaddset(&mask, SIGCHLD) == 0);
1948
1949 timeout.tv_nsec = 0;
1950 signo = -1;
1951
1982 for (;;) {
1952 for (;;) {
1983 if (sigexit_received) {
1953 switch (signo) {
1954 case SIGHUP:
1955 config_reload();
1956 break;
1957 case SIGINT:
1958 case SIGTERM:
1959 sigexit_received = true;
1984 primary_exitx(EX_OK,
1985 "Termination signal received, exiting.");
1960 primary_exitx(EX_OK,
1961 "Termination signal received, exiting.");
1962 break;
1963 default:
1964 break;
1986 }
1965 }
1987 if (sighup_received) {
1988 sighup_received = false;
1989 config_reload();
1990 }
1991 hook_check(sigchld_received);
1992 if (sigchld_received)
1993 sigchld_received = false;
1966 hook_check(signo == SIGCHLD);
1994
1995 pjdlog_debug(2, "remote_guard: Checking connections.");
1967
1968 pjdlog_debug(2, "remote_guard: Checking connections.");
1996 mtx_lock(&hio_guard_lock);
1997 timeout = KEEPALIVE_SLEEP;
1998 for (ii = 0; ii < ncomps; ii++) {
1999 if (!ISCONNECTED(res, ii)) {
2000 timeout = RECONNECT_SLEEP;
2001 break;
2002 }
2003 }
2004 now = time(NULL);
1969 now = time(NULL);
2005 if (lastcheck + timeout <= now) {
2006 timeout = KEEPALIVE_SLEEP;
2007 for (ii = 0; ii < ncomps; ii++) {
1970 if (lastcheck + RETRY_SLEEP <= now) {
1971 for (ii = 0; ii < ncomps; ii++)
2008 guard_one(res, ii);
1972 guard_one(res, ii);
2009 if (!ISCONNECTED(res, ii))
2010 timeout = RECONNECT_SLEEP;
2011 }
2012 lastcheck = now;
2013 }
1973 lastcheck = now;
1974 }
2014 /* Sleep only if a signal wasn't delivered in the meantime. */
2015 if (!sigexit_received && !sighup_received && !sigchld_received)
2016 cv_timedwait(&hio_guard_cond, &hio_guard_lock, timeout);
2017 mtx_unlock(&hio_guard_lock);
1975 timeout.tv_sec = RETRY_SLEEP;
1976 signo = sigtimedwait(&mask, NULL, &timeout);
2018 }
2019 /* NOTREACHED */
2020 return (NULL);
2021}
1977 }
1978 /* NOTREACHED */
1979 return (NULL);
1980}