nfs_commonport.c revision 240289
1/*- 2 * Copyright (c) 1989, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * This code is derived from software contributed to Berkeley by 6 * Rick Macklem at The University of Guelph. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 */ 33 34#include <sys/cdefs.h> 35__FBSDID("$FreeBSD: head/sys/fs/nfs/nfs_commonport.c 240289 2012-09-09 21:00:45Z rmacklem $"); 36 37/* 38 * Functions that need to be different for different versions of BSD 39 * kernel should be kept here, along with any global storage specific 40 * to this BSD variant. 41 */ 42#include <fs/nfs/nfsport.h> 43#include <sys/sysctl.h> 44#include <vm/vm.h> 45#include <vm/vm_object.h> 46#include <vm/vm_page.h> 47#include <vm/vm_param.h> 48#include <vm/vm_map.h> 49#include <vm/vm_kern.h> 50#include <vm/vm_extern.h> 51#include <vm/uma.h> 52 53extern int nfscl_ticks; 54extern int nfsrv_nfsuserd; 55extern struct nfssockreq nfsrv_nfsuserdsock; 56extern void (*nfsd_call_recall)(struct vnode *, int, struct ucred *, 57 struct thread *); 58extern int nfsrv_useacl; 59struct mount nfsv4root_mnt; 60int newnfs_numnfsd = 0; 61struct nfsstats newnfsstats; 62int nfs_numnfscbd = 0; 63int nfscl_debuglevel = 0; 64char nfsv4_callbackaddr[INET6_ADDRSTRLEN]; 65struct callout newnfsd_callout; 66void (*nfsd_call_servertimer)(void) = NULL; 67void (*ncl_call_invalcaches)(struct vnode *) = NULL; 68 69static int nfs_realign_test; 70static int nfs_realign_count; 71 72SYSCTL_NODE(_vfs, OID_AUTO, nfs, CTLFLAG_RW, 0, "New NFS filesystem"); 73SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_test, CTLFLAG_RW, &nfs_realign_test, 74 0, "Number of realign tests done"); 75SYSCTL_INT(_vfs_nfs, OID_AUTO, realign_count, CTLFLAG_RW, &nfs_realign_count, 76 0, "Number of mbuf realignments done"); 77SYSCTL_STRING(_vfs_nfs, OID_AUTO, callback_addr, CTLFLAG_RW, 78 nfsv4_callbackaddr, sizeof(nfsv4_callbackaddr), 79 "NFSv4 callback addr for server to use"); 80SYSCTL_INT(_vfs_nfs, OID_AUTO, debuglevel, CTLFLAG_RW, &nfscl_debuglevel, 81 0, "Debug level for new nfs client"); 82 83/* 84 * Defines for malloc 85 * (Here for FreeBSD, since they allocate storage.) 86 */ 87MALLOC_DEFINE(M_NEWNFSRVCACHE, "NFSD srvcache", "NFSD Server Request Cache"); 88MALLOC_DEFINE(M_NEWNFSDCLIENT, "NFSD V4client", "NFSD V4 Client Id"); 89MALLOC_DEFINE(M_NEWNFSDSTATE, "NFSD V4state", 90 "NFSD V4 State (Openowner, Open, Lockowner, Delegation"); 91MALLOC_DEFINE(M_NEWNFSDLOCK, "NFSD V4lock", "NFSD V4 byte range lock"); 92MALLOC_DEFINE(M_NEWNFSDLOCKFILE, "NFSD lckfile", "NFSD Open/Lock file"); 93MALLOC_DEFINE(M_NEWNFSSTRING, "NFSD string", "NFSD V4 long string"); 94MALLOC_DEFINE(M_NEWNFSUSERGROUP, "NFSD usrgroup", "NFSD V4 User/group map"); 95MALLOC_DEFINE(M_NEWNFSDREQ, "NFS req", "NFS request header"); 96MALLOC_DEFINE(M_NEWNFSFH, "NFS fh", "NFS file handle"); 97MALLOC_DEFINE(M_NEWNFSCLOWNER, "NFSCL owner", "NFSCL Open Owner"); 98MALLOC_DEFINE(M_NEWNFSCLOPEN, "NFSCL open", "NFSCL Open"); 99MALLOC_DEFINE(M_NEWNFSCLDELEG, "NFSCL deleg", "NFSCL Delegation"); 100MALLOC_DEFINE(M_NEWNFSCLCLIENT, "NFSCL client", "NFSCL Client"); 101MALLOC_DEFINE(M_NEWNFSCLLOCKOWNER, "NFSCL lckown", "NFSCL Lock Owner"); 102MALLOC_DEFINE(M_NEWNFSCLLOCK, "NFSCL lck", "NFSCL Lock"); 103MALLOC_DEFINE(M_NEWNFSV4NODE, "NEWNFSnode", "New nfs vnode"); 104MALLOC_DEFINE(M_NEWNFSDIRECTIO, "NEWdirectio", "New nfs Direct IO buffer"); 105MALLOC_DEFINE(M_NEWNFSDIROFF, "NFSCL diroffdiroff", 106 "New NFS directory offset data"); 107MALLOC_DEFINE(M_NEWNFSDROLLBACK, "NFSD rollback", 108 "New NFS local lock rollback"); 109 110/* 111 * Definition of mutex locks. 112 * newnfsd_mtx is used in nfsrvd_nfsd() to protect the nfs socket list 113 * and assorted other nfsd structures. 114 */ 115struct mtx newnfsd_mtx; 116struct mtx nfs_sockl_mutex; 117struct mtx nfs_state_mutex; 118struct mtx nfs_nameid_mutex; 119struct mtx nfs_req_mutex; 120struct mtx nfs_slock_mutex; 121 122/* local functions */ 123static int nfssvc_call(struct thread *, struct nfssvc_args *, struct ucred *); 124 125#ifdef __NO_STRICT_ALIGNMENT 126/* 127 * These architectures don't need re-alignment, so just return. 128 */ 129void 130newnfs_realign(struct mbuf **pm) 131{ 132 133 return; 134} 135#else /* !__NO_STRICT_ALIGNMENT */ 136/* 137 * newnfs_realign: 138 * 139 * Check for badly aligned mbuf data and realign by copying the unaligned 140 * portion of the data into a new mbuf chain and freeing the portions 141 * of the old chain that were replaced. 142 * 143 * We cannot simply realign the data within the existing mbuf chain 144 * because the underlying buffers may contain other rpc commands and 145 * we cannot afford to overwrite them. 146 * 147 * We would prefer to avoid this situation entirely. The situation does 148 * not occur with NFS/UDP and is supposed to only occassionally occur 149 * with TCP. Use vfs.nfs.realign_count and realign_test to check this. 150 * 151 */ 152void 153newnfs_realign(struct mbuf **pm) 154{ 155 struct mbuf *m, *n; 156 int off, space; 157 158 ++nfs_realign_test; 159 while ((m = *pm) != NULL) { 160 if ((m->m_len & 0x3) || (mtod(m, intptr_t) & 0x3)) { 161 /* 162 * NB: we can't depend on m_pkthdr.len to help us 163 * decide what to do here. May not be worth doing 164 * the m_length calculation as m_copyback will 165 * expand the mbuf chain below as needed. 166 */ 167 space = m_length(m, NULL); 168 if (space >= MINCLSIZE) { 169 /* NB: m_copyback handles space > MCLBYTES */ 170 n = m_getcl(M_WAITOK, MT_DATA, 0); 171 } else 172 n = m_get(M_WAITOK, MT_DATA); 173 if (n == NULL) 174 return; 175 /* 176 * Align the remainder of the mbuf chain. 177 */ 178 n->m_len = 0; 179 off = 0; 180 while (m != NULL) { 181 m_copyback(n, off, m->m_len, mtod(m, caddr_t)); 182 off += m->m_len; 183 m = m->m_next; 184 } 185 m_freem(*pm); 186 *pm = n; 187 ++nfs_realign_count; 188 break; 189 } 190 pm = &m->m_next; 191 } 192} 193#endif /* __NO_STRICT_ALIGNMENT */ 194 195#ifdef notdef 196static void 197nfsrv_object_create(struct vnode *vp, struct thread *td) 198{ 199 200 if (vp == NULL || vp->v_type != VREG) 201 return; 202 (void) vfs_object_create(vp, td, td->td_ucred); 203} 204#endif 205 206/* 207 * Look up a file name. Basically just initialize stuff and call namei(). 208 */ 209int 210nfsrv_lookupfilename(struct nameidata *ndp, char *fname, NFSPROC_T *p) 211{ 212 int error; 213 214 NDINIT(ndp, LOOKUP, FOLLOW | LOCKLEAF | MPSAFE, UIO_USERSPACE, fname, 215 p); 216 error = namei(ndp); 217 if (!error) { 218 NDFREE(ndp, NDF_ONLY_PNBUF); 219 } 220 return (error); 221} 222 223/* 224 * Copy NFS uid, gids to the cred structure. 225 */ 226void 227newnfs_copycred(struct nfscred *nfscr, struct ucred *cr) 228{ 229 230 KASSERT(nfscr->nfsc_ngroups >= 0, 231 ("newnfs_copycred: negative nfsc_ngroups")); 232 cr->cr_uid = nfscr->nfsc_uid; 233 crsetgroups(cr, nfscr->nfsc_ngroups, nfscr->nfsc_groups); 234} 235 236/* 237 * Map args from nfsmsleep() to msleep(). 238 */ 239int 240nfsmsleep(void *chan, void *mutex, int prio, const char *wmesg, 241 struct timespec *ts) 242{ 243 u_int64_t nsecval; 244 int error, timeo; 245 246 if (ts) { 247 timeo = hz * ts->tv_sec; 248 nsecval = (u_int64_t)ts->tv_nsec; 249 nsecval = ((nsecval * ((u_int64_t)hz)) + 500000000) / 250 1000000000; 251 timeo += (int)nsecval; 252 } else { 253 timeo = 0; 254 } 255 error = msleep(chan, (struct mtx *)mutex, prio, wmesg, timeo); 256 return (error); 257} 258 259/* 260 * Get the file system info for the server. For now, just assume FFS. 261 */ 262void 263nfsvno_getfs(struct nfsfsinfo *sip, int isdgram) 264{ 265 int pref; 266 267 /* 268 * XXX 269 * There should be file system VFS OP(s) to get this information. 270 * For now, assume ufs. 271 */ 272 if (isdgram) 273 pref = NFS_MAXDGRAMDATA; 274 else 275 pref = NFS_MAXDATA; 276 sip->fs_rtmax = NFS_MAXDATA; 277 sip->fs_rtpref = pref; 278 sip->fs_rtmult = NFS_FABLKSIZE; 279 sip->fs_wtmax = NFS_MAXDATA; 280 sip->fs_wtpref = pref; 281 sip->fs_wtmult = NFS_FABLKSIZE; 282 sip->fs_dtpref = pref; 283 sip->fs_maxfilesize = 0xffffffffffffffffull; 284 sip->fs_timedelta.tv_sec = 0; 285 sip->fs_timedelta.tv_nsec = 1; 286 sip->fs_properties = (NFSV3FSINFO_LINK | 287 NFSV3FSINFO_SYMLINK | NFSV3FSINFO_HOMOGENEOUS | 288 NFSV3FSINFO_CANSETTIME); 289} 290 291/* 292 * Do the pathconf vnode op. 293 */ 294int 295nfsvno_pathconf(struct vnode *vp, int flag, register_t *retf, 296 struct ucred *cred, struct thread *p) 297{ 298 int error; 299 300 error = VOP_PATHCONF(vp, flag, retf); 301 if (error == EOPNOTSUPP || error == EINVAL) { 302 /* 303 * Some file systems return EINVAL for name arguments not 304 * supported and some return EOPNOTSUPP for this case. 305 * So the NFSv3 Pathconf RPC doesn't fail for these cases, 306 * just fake them. 307 */ 308 switch (flag) { 309 case _PC_LINK_MAX: 310 *retf = LINK_MAX; 311 break; 312 case _PC_NAME_MAX: 313 *retf = NAME_MAX; 314 break; 315 case _PC_CHOWN_RESTRICTED: 316 *retf = 1; 317 break; 318 case _PC_NO_TRUNC: 319 *retf = 1; 320 break; 321 default: 322 /* 323 * Only happens if a _PC_xxx is added to the server, 324 * but this isn't updated. 325 */ 326 *retf = 0; 327 printf("nfsrvd pathconf flag=%d not supp\n", flag); 328 }; 329 error = 0; 330 } 331 NFSEXITCODE(error); 332 return (error); 333} 334 335/* Fake nfsrv_atroot. Just return 0 */ 336int 337nfsrv_atroot(struct vnode *vp, long *retp) 338{ 339 340 return (0); 341} 342 343/* 344 * Set the credentials to refer to root. 345 * If only the various BSDen could agree on whether cr_gid is a separate 346 * field or cr_groups[0]... 347 */ 348void 349newnfs_setroot(struct ucred *cred) 350{ 351 352 cred->cr_uid = 0; 353 cred->cr_groups[0] = 0; 354 cred->cr_ngroups = 1; 355} 356 357/* 358 * Get the client credential. Used for Renew and recovery. 359 */ 360struct ucred * 361newnfs_getcred(void) 362{ 363 struct ucred *cred; 364 struct thread *td = curthread; 365 366 cred = crdup(td->td_ucred); 367 newnfs_setroot(cred); 368 return (cred); 369} 370 371/* 372 * Nfs timer routine 373 * Call the nfsd's timer function once/sec. 374 */ 375void 376newnfs_timer(void *arg) 377{ 378 static time_t lasttime = 0; 379 /* 380 * Call the server timer, if set up. 381 * The argument indicates if it is the next second and therefore 382 * leases should be checked. 383 */ 384 if (lasttime != NFSD_MONOSEC) { 385 lasttime = NFSD_MONOSEC; 386 if (nfsd_call_servertimer != NULL) 387 (*nfsd_call_servertimer)(); 388 } 389 callout_reset(&newnfsd_callout, nfscl_ticks, newnfs_timer, NULL); 390} 391 392 393/* 394 * Sleep for a short period of time unless errval == NFSERR_GRACE, where 395 * the sleep should be for 5 seconds. 396 * Since lbolt doesn't exist in FreeBSD-CURRENT, just use a timeout on 397 * an event that never gets a wakeup. Only return EINTR or 0. 398 */ 399int 400nfs_catnap(int prio, int errval, const char *wmesg) 401{ 402 static int non_event; 403 int ret; 404 405 if (errval == NFSERR_GRACE) 406 ret = tsleep(&non_event, prio, wmesg, 5 * hz); 407 else 408 ret = tsleep(&non_event, prio, wmesg, 1); 409 if (ret != EINTR) 410 ret = 0; 411 return (ret); 412} 413 414/* 415 * Get referral. For now, just fail. 416 */ 417struct nfsreferral * 418nfsv4root_getreferral(struct vnode *vp, struct vnode *dvp, u_int32_t fileno) 419{ 420 421 return (NULL); 422} 423 424static int 425nfssvc_nfscommon(struct thread *td, struct nfssvc_args *uap) 426{ 427 int error; 428 429 error = nfssvc_call(td, uap, td->td_ucred); 430 NFSEXITCODE(error); 431 return (error); 432} 433 434static int 435nfssvc_call(struct thread *p, struct nfssvc_args *uap, struct ucred *cred) 436{ 437 int error = EINVAL; 438 struct nfsd_idargs nid; 439 440 if (uap->flag & NFSSVC_IDNAME) { 441 error = copyin(uap->argp, (caddr_t)&nid, sizeof (nid)); 442 if (error) 443 goto out; 444 error = nfssvc_idname(&nid); 445 goto out; 446 } else if (uap->flag & NFSSVC_GETSTATS) { 447 error = copyout(&newnfsstats, 448 CAST_USER_ADDR_T(uap->argp), sizeof (newnfsstats)); 449 if (error == 0) { 450 if ((uap->flag & NFSSVC_ZEROCLTSTATS) != 0) { 451 newnfsstats.attrcache_hits = 0; 452 newnfsstats.attrcache_misses = 0; 453 newnfsstats.lookupcache_hits = 0; 454 newnfsstats.lookupcache_misses = 0; 455 newnfsstats.direofcache_hits = 0; 456 newnfsstats.direofcache_misses = 0; 457 newnfsstats.accesscache_hits = 0; 458 newnfsstats.accesscache_misses = 0; 459 newnfsstats.biocache_reads = 0; 460 newnfsstats.read_bios = 0; 461 newnfsstats.read_physios = 0; 462 newnfsstats.biocache_writes = 0; 463 newnfsstats.write_bios = 0; 464 newnfsstats.write_physios = 0; 465 newnfsstats.biocache_readlinks = 0; 466 newnfsstats.readlink_bios = 0; 467 newnfsstats.biocache_readdirs = 0; 468 newnfsstats.readdir_bios = 0; 469 newnfsstats.rpcretries = 0; 470 newnfsstats.rpcrequests = 0; 471 newnfsstats.rpctimeouts = 0; 472 newnfsstats.rpcunexpected = 0; 473 newnfsstats.rpcinvalid = 0; 474 bzero(newnfsstats.rpccnt, 475 sizeof(newnfsstats.rpccnt)); 476 } 477 if ((uap->flag & NFSSVC_ZEROSRVSTATS) != 0) { 478 newnfsstats.srvrpc_errs = 0; 479 newnfsstats.srv_errs = 0; 480 newnfsstats.srvcache_inproghits = 0; 481 newnfsstats.srvcache_idemdonehits = 0; 482 newnfsstats.srvcache_nonidemdonehits = 0; 483 newnfsstats.srvcache_misses = 0; 484 newnfsstats.srvcache_tcppeak = 0; 485 newnfsstats.srvclients = 0; 486 newnfsstats.srvopenowners = 0; 487 newnfsstats.srvopens = 0; 488 newnfsstats.srvlockowners = 0; 489 newnfsstats.srvlocks = 0; 490 newnfsstats.srvdelegates = 0; 491 newnfsstats.clopenowners = 0; 492 newnfsstats.clopens = 0; 493 newnfsstats.cllockowners = 0; 494 newnfsstats.cllocks = 0; 495 newnfsstats.cldelegates = 0; 496 newnfsstats.cllocalopenowners = 0; 497 newnfsstats.cllocalopens = 0; 498 newnfsstats.cllocallockowners = 0; 499 newnfsstats.cllocallocks = 0; 500 bzero(newnfsstats.srvrpccnt, 501 sizeof(newnfsstats.srvrpccnt)); 502 bzero(newnfsstats.cbrpccnt, 503 sizeof(newnfsstats.cbrpccnt)); 504 } 505 } 506 goto out; 507 } else if (uap->flag & NFSSVC_NFSUSERDPORT) { 508 u_short sockport; 509 510 error = copyin(uap->argp, (caddr_t)&sockport, 511 sizeof (u_short)); 512 if (!error) 513 error = nfsrv_nfsuserdport(sockport, p); 514 } else if (uap->flag & NFSSVC_NFSUSERDDELPORT) { 515 nfsrv_nfsuserddelport(); 516 error = 0; 517 } 518 519out: 520 NFSEXITCODE(error); 521 return (error); 522} 523 524/* 525 * called by all three modevent routines, so that it gets things 526 * initialized soon enough. 527 */ 528void 529newnfs_portinit(void) 530{ 531 static int inited = 0; 532 533 if (inited) 534 return; 535 inited = 1; 536 /* Initialize SMP locks used by both client and server. */ 537 mtx_init(&newnfsd_mtx, "newnfsd_mtx", NULL, MTX_DEF); 538 mtx_init(&nfs_state_mutex, "nfs_state_mutex", NULL, MTX_DEF); 539} 540 541/* 542 * Determine if the file system supports NFSv4 ACLs. 543 * Return 1 if it does, 0 otherwise. 544 */ 545int 546nfs_supportsnfsv4acls(struct vnode *vp) 547{ 548 int error; 549 register_t retval; 550 551 ASSERT_VOP_LOCKED(vp, "nfs supports nfsv4acls"); 552 553 if (nfsrv_useacl == 0) 554 return (0); 555 error = VOP_PATHCONF(vp, _PC_ACL_NFS4, &retval); 556 if (error == 0 && retval != 0) 557 return (1); 558 return (0); 559} 560 561extern int (*nfsd_call_nfscommon)(struct thread *, struct nfssvc_args *); 562 563/* 564 * Called once to initialize data structures... 565 */ 566static int 567nfscommon_modevent(module_t mod, int type, void *data) 568{ 569 int error = 0; 570 static int loaded = 0; 571 572 switch (type) { 573 case MOD_LOAD: 574 if (loaded) 575 goto out; 576 newnfs_portinit(); 577 mtx_init(&nfs_nameid_mutex, "nfs_nameid_mutex", NULL, MTX_DEF); 578 mtx_init(&nfs_sockl_mutex, "nfs_sockl_mutex", NULL, MTX_DEF); 579 mtx_init(&nfs_slock_mutex, "nfs_slock_mutex", NULL, MTX_DEF); 580 mtx_init(&nfs_req_mutex, "nfs_req_mutex", NULL, MTX_DEF); 581 mtx_init(&nfsrv_nfsuserdsock.nr_mtx, "nfsuserd", NULL, 582 MTX_DEF); 583 callout_init(&newnfsd_callout, CALLOUT_MPSAFE); 584 newnfs_init(); 585 nfsd_call_nfscommon = nfssvc_nfscommon; 586 loaded = 1; 587 break; 588 589 case MOD_UNLOAD: 590 if (newnfs_numnfsd != 0 || nfsrv_nfsuserd != 0 || 591 nfs_numnfscbd != 0) { 592 error = EBUSY; 593 break; 594 } 595 596 nfsd_call_nfscommon = NULL; 597 callout_drain(&newnfsd_callout); 598 /* and get rid of the mutexes */ 599 mtx_destroy(&nfs_nameid_mutex); 600 mtx_destroy(&newnfsd_mtx); 601 mtx_destroy(&nfs_state_mutex); 602 mtx_destroy(&nfs_sockl_mutex); 603 mtx_destroy(&nfs_slock_mutex); 604 mtx_destroy(&nfs_req_mutex); 605 mtx_destroy(&nfsrv_nfsuserdsock.nr_mtx); 606 loaded = 0; 607 break; 608 default: 609 error = EOPNOTSUPP; 610 break; 611 } 612 613out: 614 NFSEXITCODE(error); 615 return error; 616} 617static moduledata_t nfscommon_mod = { 618 "nfscommon", 619 nfscommon_modevent, 620 NULL, 621}; 622DECLARE_MODULE(nfscommon, nfscommon_mod, SI_SUB_VFS, SI_ORDER_ANY); 623 624/* So that loader and kldload(2) can find us, wherever we are.. */ 625MODULE_VERSION(nfscommon, 1); 626MODULE_DEPEND(nfscommon, nfssvc, 1, 1, 1); 627MODULE_DEPEND(nfscommon, krpc, 1, 1, 1); 628 629