1/* Cluster IP hashmark target 2 * (C) 2003-2004 by Harald Welte <laforge@netfilter.org> 3 * based on ideas of Fabio Olive Leite <olive@unixforge.org> 4 * 5 * Development of this code funded by SuSE Linux AG, http://www.suse.com/ 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 */ 12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 13#include <linux/module.h> 14#include <linux/proc_fs.h> 15#include <linux/jhash.h> 16#include <linux/bitops.h> 17#include <linux/skbuff.h> 18#include <linux/slab.h> 19#include <linux/ip.h> 20#include <linux/tcp.h> 21#include <linux/udp.h> 22#include <linux/icmp.h> 23#include <linux/if_arp.h> 24#include <linux/seq_file.h> 25#include <linux/netfilter_arp.h> 26#include <linux/netfilter/x_tables.h> 27#include <linux/netfilter_ipv4/ip_tables.h> 28#include <linux/netfilter_ipv4/ipt_CLUSTERIP.h> 29#include <net/netfilter/nf_conntrack.h> 30#include <net/net_namespace.h> 31#include <net/checksum.h> 32 33#define CLUSTERIP_VERSION "0.8" 34 35MODULE_LICENSE("GPL"); 36MODULE_AUTHOR("Harald Welte <laforge@netfilter.org>"); 37MODULE_DESCRIPTION("Xtables: CLUSTERIP target"); 38 39struct clusterip_config { 40 struct list_head list; /* list of all configs */ 41 atomic_t refcount; /* reference count */ 42 atomic_t entries; /* number of entries/rules 43 * referencing us */ 44 45 __be32 clusterip; /* the IP address */ 46 u_int8_t clustermac[ETH_ALEN]; /* the MAC address */ 47 struct net_device *dev; /* device */ 48 u_int16_t num_total_nodes; /* total number of nodes */ 49 unsigned long local_nodes; /* node number array */ 50 51#ifdef CONFIG_PROC_FS 52 struct proc_dir_entry *pde; /* proc dir entry */ 53#endif 54 enum clusterip_hashmode hash_mode; /* which hashing mode */ 55 u_int32_t hash_initval; /* hash initialization */ 56 struct rcu_head rcu; 57}; 58 59static LIST_HEAD(clusterip_configs); 60 61/* clusterip_lock protects the clusterip_configs list */ 62static DEFINE_SPINLOCK(clusterip_lock); 63 64#ifdef CONFIG_PROC_FS 65static const struct file_operations clusterip_proc_fops; 66static struct proc_dir_entry *clusterip_procdir; 67#endif 68 69static inline void 70clusterip_config_get(struct clusterip_config *c) 71{ 72 atomic_inc(&c->refcount); 73} 74 75 76static void clusterip_config_rcu_free(struct rcu_head *head) 77{ 78 kfree(container_of(head, struct clusterip_config, rcu)); 79} 80 81static inline void 82clusterip_config_put(struct clusterip_config *c) 83{ 84 if (atomic_dec_and_test(&c->refcount)) 85 call_rcu_bh(&c->rcu, clusterip_config_rcu_free); 86} 87 88/* decrease the count of entries using/referencing this config. If last 89 * entry(rule) is removed, remove the config from lists, but don't free it 90 * yet, since proc-files could still be holding references */ 91static inline void 92clusterip_config_entry_put(struct clusterip_config *c) 93{ 94 local_bh_disable(); 95 if (atomic_dec_and_lock(&c->entries, &clusterip_lock)) { 96 list_del_rcu(&c->list); 97 spin_unlock(&clusterip_lock); 98 local_bh_enable(); 99 100 dev_mc_del(c->dev, c->clustermac); 101 dev_put(c->dev); 102 103 /* In case anyone still accesses the file, the open/close 104 * functions are also incrementing the refcount on their own, 105 * so it's safe to remove the entry even if it's in use. */ 106#ifdef CONFIG_PROC_FS 107 remove_proc_entry(c->pde->name, c->pde->parent); 108#endif 109 return; 110 } 111 local_bh_enable(); 112} 113 114static struct clusterip_config * 115__clusterip_config_find(__be32 clusterip) 116{ 117 struct clusterip_config *c; 118 119 list_for_each_entry_rcu(c, &clusterip_configs, list) { 120 if (c->clusterip == clusterip) 121 return c; 122 } 123 124 return NULL; 125} 126 127static inline struct clusterip_config * 128clusterip_config_find_get(__be32 clusterip, int entry) 129{ 130 struct clusterip_config *c; 131 132 rcu_read_lock_bh(); 133 c = __clusterip_config_find(clusterip); 134 if (c) { 135 if (unlikely(!atomic_inc_not_zero(&c->refcount))) 136 c = NULL; 137 else if (entry) 138 atomic_inc(&c->entries); 139 } 140 rcu_read_unlock_bh(); 141 142 return c; 143} 144 145static void 146clusterip_config_init_nodelist(struct clusterip_config *c, 147 const struct ipt_clusterip_tgt_info *i) 148{ 149 int n; 150 151 for (n = 0; n < i->num_local_nodes; n++) 152 set_bit(i->local_nodes[n] - 1, &c->local_nodes); 153} 154 155static struct clusterip_config * 156clusterip_config_init(const struct ipt_clusterip_tgt_info *i, __be32 ip, 157 struct net_device *dev) 158{ 159 struct clusterip_config *c; 160 161 c = kzalloc(sizeof(*c), GFP_ATOMIC); 162 if (!c) 163 return NULL; 164 165 c->dev = dev; 166 c->clusterip = ip; 167 memcpy(&c->clustermac, &i->clustermac, ETH_ALEN); 168 c->num_total_nodes = i->num_total_nodes; 169 clusterip_config_init_nodelist(c, i); 170 c->hash_mode = i->hash_mode; 171 c->hash_initval = i->hash_initval; 172 atomic_set(&c->refcount, 1); 173 atomic_set(&c->entries, 1); 174 175#ifdef CONFIG_PROC_FS 176 { 177 char buffer[16]; 178 179 /* create proc dir entry */ 180 sprintf(buffer, "%pI4", &ip); 181 c->pde = proc_create_data(buffer, S_IWUSR|S_IRUSR, 182 clusterip_procdir, 183 &clusterip_proc_fops, c); 184 if (!c->pde) { 185 kfree(c); 186 return NULL; 187 } 188 } 189#endif 190 191 spin_lock_bh(&clusterip_lock); 192 list_add_rcu(&c->list, &clusterip_configs); 193 spin_unlock_bh(&clusterip_lock); 194 195 return c; 196} 197 198#ifdef CONFIG_PROC_FS 199static int 200clusterip_add_node(struct clusterip_config *c, u_int16_t nodenum) 201{ 202 203 if (nodenum == 0 || 204 nodenum > c->num_total_nodes) 205 return 1; 206 207 /* check if we already have this number in our bitfield */ 208 if (test_and_set_bit(nodenum - 1, &c->local_nodes)) 209 return 1; 210 211 return 0; 212} 213 214static bool 215clusterip_del_node(struct clusterip_config *c, u_int16_t nodenum) 216{ 217 if (nodenum == 0 || 218 nodenum > c->num_total_nodes) 219 return true; 220 221 if (test_and_clear_bit(nodenum - 1, &c->local_nodes)) 222 return false; 223 224 return true; 225} 226#endif 227 228static inline u_int32_t 229clusterip_hashfn(const struct sk_buff *skb, 230 const struct clusterip_config *config) 231{ 232 const struct iphdr *iph = ip_hdr(skb); 233 unsigned long hashval; 234 u_int16_t sport, dport; 235 const u_int16_t *ports; 236 237 switch (iph->protocol) { 238 case IPPROTO_TCP: 239 case IPPROTO_UDP: 240 case IPPROTO_UDPLITE: 241 case IPPROTO_SCTP: 242 case IPPROTO_DCCP: 243 case IPPROTO_ICMP: 244 ports = (const void *)iph+iph->ihl*4; 245 sport = ports[0]; 246 dport = ports[1]; 247 break; 248 default: 249 if (net_ratelimit()) 250 pr_info("unknown protocol %u\n", iph->protocol); 251 sport = dport = 0; 252 } 253 254 switch (config->hash_mode) { 255 case CLUSTERIP_HASHMODE_SIP: 256 hashval = jhash_1word(ntohl(iph->saddr), 257 config->hash_initval); 258 break; 259 case CLUSTERIP_HASHMODE_SIP_SPT: 260 hashval = jhash_2words(ntohl(iph->saddr), sport, 261 config->hash_initval); 262 break; 263 case CLUSTERIP_HASHMODE_SIP_SPT_DPT: 264 hashval = jhash_3words(ntohl(iph->saddr), sport, dport, 265 config->hash_initval); 266 break; 267 default: 268 /* to make gcc happy */ 269 hashval = 0; 270 /* This cannot happen, unless the check function wasn't called 271 * at rule load time */ 272 pr_info("unknown mode %u\n", config->hash_mode); 273 BUG(); 274 break; 275 } 276 277 /* node numbers are 1..n, not 0..n */ 278 return (((u64)hashval * config->num_total_nodes) >> 32) + 1; 279} 280 281static inline int 282clusterip_responsible(const struct clusterip_config *config, u_int32_t hash) 283{ 284 return test_bit(hash - 1, &config->local_nodes); 285} 286 287/*********************************************************************** 288 * IPTABLES TARGET 289 ***********************************************************************/ 290 291static unsigned int 292clusterip_tg(struct sk_buff *skb, const struct xt_action_param *par) 293{ 294 const struct ipt_clusterip_tgt_info *cipinfo = par->targinfo; 295 struct nf_conn *ct; 296 enum ip_conntrack_info ctinfo; 297 u_int32_t hash; 298 299 /* don't need to clusterip_config_get() here, since refcount 300 * is only decremented by destroy() - and ip_tables guarantees 301 * that the ->target() function isn't called after ->destroy() */ 302 303 ct = nf_ct_get(skb, &ctinfo); 304 if (ct == NULL) { 305 pr_info("no conntrack!\n"); 306 return NF_DROP; 307 } 308 309 /* special case: ICMP error handling. conntrack distinguishes between 310 * error messages (RELATED) and information requests (see below) */ 311 if (ip_hdr(skb)->protocol == IPPROTO_ICMP && 312 (ctinfo == IP_CT_RELATED || 313 ctinfo == IP_CT_RELATED + IP_CT_IS_REPLY)) 314 return XT_CONTINUE; 315 316 /* ip_conntrack_icmp guarantees us that we only have ICMP_ECHO, 317 * TIMESTAMP, INFO_REQUEST or ADDRESS type icmp packets from here 318 * on, which all have an ID field [relevant for hashing]. */ 319 320 hash = clusterip_hashfn(skb, cipinfo->config); 321 322 switch (ctinfo) { 323 case IP_CT_NEW: 324 ct->mark = hash; 325 break; 326 case IP_CT_RELATED: 327 case IP_CT_RELATED+IP_CT_IS_REPLY: 328 case IP_CT_ESTABLISHED: 329 case IP_CT_ESTABLISHED+IP_CT_IS_REPLY: 330 break; 331 default: 332 break; 333 } 334 335#ifdef DEBUG 336 nf_ct_dump_tuple_ip(&ct->tuplehash[IP_CT_DIR_ORIGINAL].tuple); 337#endif 338 pr_debug("hash=%u ct_hash=%u ", hash, ct->mark); 339 if (!clusterip_responsible(cipinfo->config, hash)) { 340 pr_debug("not responsible\n"); 341 return NF_DROP; 342 } 343 pr_debug("responsible\n"); 344 345 /* despite being received via linklayer multicast, this is 346 * actually a unicast IP packet. TCP doesn't like PACKET_MULTICAST */ 347 skb->pkt_type = PACKET_HOST; 348 349 return XT_CONTINUE; 350} 351 352static int clusterip_tg_check(const struct xt_tgchk_param *par) 353{ 354 struct ipt_clusterip_tgt_info *cipinfo = par->targinfo; 355 const struct ipt_entry *e = par->entryinfo; 356 struct clusterip_config *config; 357 int ret; 358 359 if (cipinfo->hash_mode != CLUSTERIP_HASHMODE_SIP && 360 cipinfo->hash_mode != CLUSTERIP_HASHMODE_SIP_SPT && 361 cipinfo->hash_mode != CLUSTERIP_HASHMODE_SIP_SPT_DPT) { 362 pr_info("unknown mode %u\n", cipinfo->hash_mode); 363 return -EINVAL; 364 365 } 366 if (e->ip.dmsk.s_addr != htonl(0xffffffff) || 367 e->ip.dst.s_addr == 0) { 368 pr_info("Please specify destination IP\n"); 369 return -EINVAL; 370 } 371 372 373 config = clusterip_config_find_get(e->ip.dst.s_addr, 1); 374 if (!config) { 375 if (!(cipinfo->flags & CLUSTERIP_FLAG_NEW)) { 376 pr_info("no config found for %pI4, need 'new'\n", 377 &e->ip.dst.s_addr); 378 return -EINVAL; 379 } else { 380 struct net_device *dev; 381 382 if (e->ip.iniface[0] == '\0') { 383 pr_info("Please specify an interface name\n"); 384 return -EINVAL; 385 } 386 387 dev = dev_get_by_name(&init_net, e->ip.iniface); 388 if (!dev) { 389 pr_info("no such interface %s\n", 390 e->ip.iniface); 391 return -ENOENT; 392 } 393 394 config = clusterip_config_init(cipinfo, 395 e->ip.dst.s_addr, dev); 396 if (!config) { 397 pr_info("cannot allocate config\n"); 398 dev_put(dev); 399 return -ENOMEM; 400 } 401 dev_mc_add(config->dev, config->clustermac); 402 } 403 } 404 cipinfo->config = config; 405 406 ret = nf_ct_l3proto_try_module_get(par->family); 407 if (ret < 0) 408 pr_info("cannot load conntrack support for proto=%u\n", 409 par->family); 410 return ret; 411} 412 413/* drop reference count of cluster config when rule is deleted */ 414static void clusterip_tg_destroy(const struct xt_tgdtor_param *par) 415{ 416 const struct ipt_clusterip_tgt_info *cipinfo = par->targinfo; 417 418 /* if no more entries are referencing the config, remove it 419 * from the list and destroy the proc entry */ 420 clusterip_config_entry_put(cipinfo->config); 421 422 clusterip_config_put(cipinfo->config); 423 424 nf_ct_l3proto_module_put(par->family); 425} 426 427#ifdef CONFIG_COMPAT 428struct compat_ipt_clusterip_tgt_info 429{ 430 u_int32_t flags; 431 u_int8_t clustermac[6]; 432 u_int16_t num_total_nodes; 433 u_int16_t num_local_nodes; 434 u_int16_t local_nodes[CLUSTERIP_MAX_NODES]; 435 u_int32_t hash_mode; 436 u_int32_t hash_initval; 437 compat_uptr_t config; 438}; 439#endif /* CONFIG_COMPAT */ 440 441static struct xt_target clusterip_tg_reg __read_mostly = { 442 .name = "CLUSTERIP", 443 .family = NFPROTO_IPV4, 444 .target = clusterip_tg, 445 .checkentry = clusterip_tg_check, 446 .destroy = clusterip_tg_destroy, 447 .targetsize = sizeof(struct ipt_clusterip_tgt_info), 448#ifdef CONFIG_COMPAT 449 .compatsize = sizeof(struct compat_ipt_clusterip_tgt_info), 450#endif /* CONFIG_COMPAT */ 451 .me = THIS_MODULE 452}; 453 454 455/*********************************************************************** 456 * ARP MANGLING CODE 457 ***********************************************************************/ 458 459/* hardcoded for 48bit ethernet and 32bit ipv4 addresses */ 460struct arp_payload { 461 u_int8_t src_hw[ETH_ALEN]; 462 __be32 src_ip; 463 u_int8_t dst_hw[ETH_ALEN]; 464 __be32 dst_ip; 465} __packed; 466 467#ifdef DEBUG 468static void arp_print(struct arp_payload *payload) 469{ 470#define HBUFFERLEN 30 471 char hbuffer[HBUFFERLEN]; 472 int j,k; 473 474 for (k=0, j=0; k < HBUFFERLEN-3 && j < ETH_ALEN; j++) { 475 hbuffer[k++] = hex_asc_hi(payload->src_hw[j]); 476 hbuffer[k++] = hex_asc_lo(payload->src_hw[j]); 477 hbuffer[k++]=':'; 478 } 479 hbuffer[--k]='\0'; 480 481 pr_debug("src %pI4@%s, dst %pI4\n", 482 &payload->src_ip, hbuffer, &payload->dst_ip); 483} 484#endif 485 486static unsigned int 487arp_mangle(unsigned int hook, 488 struct sk_buff *skb, 489 const struct net_device *in, 490 const struct net_device *out, 491 int (*okfn)(struct sk_buff *)) 492{ 493 struct arphdr *arp = arp_hdr(skb); 494 struct arp_payload *payload; 495 struct clusterip_config *c; 496 497 /* we don't care about non-ethernet and non-ipv4 ARP */ 498 if (arp->ar_hrd != htons(ARPHRD_ETHER) || 499 arp->ar_pro != htons(ETH_P_IP) || 500 arp->ar_pln != 4 || arp->ar_hln != ETH_ALEN) 501 return NF_ACCEPT; 502 503 /* we only want to mangle arp requests and replies */ 504 if (arp->ar_op != htons(ARPOP_REPLY) && 505 arp->ar_op != htons(ARPOP_REQUEST)) 506 return NF_ACCEPT; 507 508 payload = (void *)(arp+1); 509 510 /* if there is no clusterip configuration for the arp reply's 511 * source ip, we don't want to mangle it */ 512 c = clusterip_config_find_get(payload->src_ip, 0); 513 if (!c) 514 return NF_ACCEPT; 515 516 /* normally the linux kernel always replies to arp queries of 517 * addresses on different interfacs. However, in the CLUSTERIP case 518 * this wouldn't work, since we didn't subscribe the mcast group on 519 * other interfaces */ 520 if (c->dev != out) { 521 pr_debug("not mangling arp reply on different " 522 "interface: cip'%s'-skb'%s'\n", 523 c->dev->name, out->name); 524 clusterip_config_put(c); 525 return NF_ACCEPT; 526 } 527 528 /* mangle reply hardware address */ 529 memcpy(payload->src_hw, c->clustermac, arp->ar_hln); 530 531#ifdef DEBUG 532 pr_debug("mangled arp reply: "); 533 arp_print(payload); 534#endif 535 536 clusterip_config_put(c); 537 538 return NF_ACCEPT; 539} 540 541static struct nf_hook_ops cip_arp_ops __read_mostly = { 542 .hook = arp_mangle, 543 .pf = NFPROTO_ARP, 544 .hooknum = NF_ARP_OUT, 545 .priority = -1 546}; 547 548/*********************************************************************** 549 * PROC DIR HANDLING 550 ***********************************************************************/ 551 552#ifdef CONFIG_PROC_FS 553 554struct clusterip_seq_position { 555 unsigned int pos; /* position */ 556 unsigned int weight; /* number of bits set == size */ 557 unsigned int bit; /* current bit */ 558 unsigned long val; /* current value */ 559}; 560 561static void *clusterip_seq_start(struct seq_file *s, loff_t *pos) 562{ 563 struct clusterip_config *c = s->private; 564 unsigned int weight; 565 u_int32_t local_nodes; 566 struct clusterip_seq_position *idx; 567 568 local_nodes = c->local_nodes; 569 weight = hweight32(local_nodes); 570 if (*pos >= weight) 571 return NULL; 572 573 idx = kmalloc(sizeof(struct clusterip_seq_position), GFP_KERNEL); 574 if (!idx) 575 return ERR_PTR(-ENOMEM); 576 577 idx->pos = *pos; 578 idx->weight = weight; 579 idx->bit = ffs(local_nodes); 580 idx->val = local_nodes; 581 clear_bit(idx->bit - 1, &idx->val); 582 583 return idx; 584} 585 586static void *clusterip_seq_next(struct seq_file *s, void *v, loff_t *pos) 587{ 588 struct clusterip_seq_position *idx = v; 589 590 *pos = ++idx->pos; 591 if (*pos >= idx->weight) { 592 kfree(v); 593 return NULL; 594 } 595 idx->bit = ffs(idx->val); 596 clear_bit(idx->bit - 1, &idx->val); 597 return idx; 598} 599 600static void clusterip_seq_stop(struct seq_file *s, void *v) 601{ 602 if (!IS_ERR(v)) 603 kfree(v); 604} 605 606static int clusterip_seq_show(struct seq_file *s, void *v) 607{ 608 struct clusterip_seq_position *idx = v; 609 610 if (idx->pos != 0) 611 seq_putc(s, ','); 612 613 seq_printf(s, "%u", idx->bit); 614 615 if (idx->pos == idx->weight - 1) 616 seq_putc(s, '\n'); 617 618 return 0; 619} 620 621static const struct seq_operations clusterip_seq_ops = { 622 .start = clusterip_seq_start, 623 .next = clusterip_seq_next, 624 .stop = clusterip_seq_stop, 625 .show = clusterip_seq_show, 626}; 627 628static int clusterip_proc_open(struct inode *inode, struct file *file) 629{ 630 int ret = seq_open(file, &clusterip_seq_ops); 631 632 if (!ret) { 633 struct seq_file *sf = file->private_data; 634 struct clusterip_config *c = PDE(inode)->data; 635 636 sf->private = c; 637 638 clusterip_config_get(c); 639 } 640 641 return ret; 642} 643 644static int clusterip_proc_release(struct inode *inode, struct file *file) 645{ 646 struct clusterip_config *c = PDE(inode)->data; 647 int ret; 648 649 ret = seq_release(inode, file); 650 651 if (!ret) 652 clusterip_config_put(c); 653 654 return ret; 655} 656 657static ssize_t clusterip_proc_write(struct file *file, const char __user *input, 658 size_t size, loff_t *ofs) 659{ 660 struct clusterip_config *c = PDE(file->f_path.dentry->d_inode)->data; 661#define PROC_WRITELEN 10 662 char buffer[PROC_WRITELEN+1]; 663 unsigned long nodenum; 664 665 if (copy_from_user(buffer, input, PROC_WRITELEN)) 666 return -EFAULT; 667 668 if (*buffer == '+') { 669 nodenum = simple_strtoul(buffer+1, NULL, 10); 670 if (clusterip_add_node(c, nodenum)) 671 return -ENOMEM; 672 } else if (*buffer == '-') { 673 nodenum = simple_strtoul(buffer+1, NULL,10); 674 if (clusterip_del_node(c, nodenum)) 675 return -ENOENT; 676 } else 677 return -EIO; 678 679 return size; 680} 681 682static const struct file_operations clusterip_proc_fops = { 683 .owner = THIS_MODULE, 684 .open = clusterip_proc_open, 685 .read = seq_read, 686 .write = clusterip_proc_write, 687 .llseek = seq_lseek, 688 .release = clusterip_proc_release, 689}; 690 691#endif /* CONFIG_PROC_FS */ 692 693static int __init clusterip_tg_init(void) 694{ 695 int ret; 696 697 ret = xt_register_target(&clusterip_tg_reg); 698 if (ret < 0) 699 return ret; 700 701 ret = nf_register_hook(&cip_arp_ops); 702 if (ret < 0) 703 goto cleanup_target; 704 705#ifdef CONFIG_PROC_FS 706 clusterip_procdir = proc_mkdir("ipt_CLUSTERIP", init_net.proc_net); 707 if (!clusterip_procdir) { 708 pr_err("Unable to proc dir entry\n"); 709 ret = -ENOMEM; 710 goto cleanup_hook; 711 } 712#endif /* CONFIG_PROC_FS */ 713 714 pr_info("ClusterIP Version %s loaded successfully\n", 715 CLUSTERIP_VERSION); 716 return 0; 717 718#ifdef CONFIG_PROC_FS 719cleanup_hook: 720 nf_unregister_hook(&cip_arp_ops); 721#endif /* CONFIG_PROC_FS */ 722cleanup_target: 723 xt_unregister_target(&clusterip_tg_reg); 724 return ret; 725} 726 727static void __exit clusterip_tg_exit(void) 728{ 729 pr_info("ClusterIP Version %s unloading\n", CLUSTERIP_VERSION); 730#ifdef CONFIG_PROC_FS 731 remove_proc_entry(clusterip_procdir->name, clusterip_procdir->parent); 732#endif 733 nf_unregister_hook(&cip_arp_ops); 734 xt_unregister_target(&clusterip_tg_reg); 735 736 /* Wait for completion of call_rcu_bh()'s (clusterip_config_rcu_free) */ 737 rcu_barrier_bh(); 738} 739 740module_init(clusterip_tg_init); 741module_exit(clusterip_tg_exit); 742