1// format.cpp -- 2// $Id: format.cpp 1230 2007-03-09 15:58:53Z jcw $ 3// This is part of Metakit, the homepage is http://www.equi4.com/metakit.html 4 5/** @file 6 * Format handlers deal with the representation of data 7 */ 8 9#include "header.h" 10#include "handler.h" 11#include "column.h" 12#include "format.h" 13#include "persist.h" 14 15///////////////////////////////////////////////////////////////////////////// 16 17class c4_FormatHandler: public c4_Handler { 18 c4_HandlerSeq &_owner; 19 20 public: 21 c4_FormatHandler(const c4_Property &prop_, c4_HandlerSeq &owner_); 22 virtual ~c4_FormatHandler(); 23 24 virtual bool IsPersistent()const; 25 26 protected: 27 c4_HandlerSeq &Owner()const; 28}; 29 30///////////////////////////////////////////////////////////////////////////// 31// c4_FormatHandler 32 33c4_FormatHandler::c4_FormatHandler(const c4_Property &prop_, c4_HandlerSeq 34 &owner_): c4_Handler(prop_), _owner(owner_){} 35 36c4_FormatHandler::~c4_FormatHandler(){} 37 38d4_inline c4_HandlerSeq &c4_FormatHandler::Owner()const { 39 return _owner; 40} 41 42bool c4_FormatHandler::IsPersistent()const { 43 return _owner.Persist() != 0; 44} 45 46///////////////////////////////////////////////////////////////////////////// 47 48class c4_FormatX: public c4_FormatHandler { 49 public: 50 c4_FormatX(const c4_Property &prop_, c4_HandlerSeq &seq_, int width_ = 51 sizeof(t4_i32)); 52 53 virtual void Define(int, const t4_byte **); 54 virtual void OldDefine(char type_, c4_Persist &); 55 virtual void FlipBytes(); 56 57 virtual int ItemSize(int index_); 58 virtual const void *Get(int index_, int &length_); 59 virtual void Set(int index_, const c4_Bytes &buf_); 60 61 virtual void Insert(int index_, const c4_Bytes &buf_, int count_); 62 virtual void Remove(int index_, int count_); 63 64 virtual void Commit(c4_SaveContext &ar_); 65 66 virtual void Unmapped(); 67 68 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 69 70 protected: 71 c4_ColOfInts _data; 72}; 73 74///////////////////////////////////////////////////////////////////////////// 75 76c4_FormatX::c4_FormatX(const c4_Property &p_, c4_HandlerSeq &s_, int w_): 77 c4_FormatHandler(p_, s_), _data(s_.Persist(), w_){} 78 79int c4_FormatX::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 80 return c4_ColOfInts::DoCompare(b1_, b2_); 81} 82 83void c4_FormatX::Commit(c4_SaveContext &ar_) { 84 _data.FixSize(true); 85 ar_.CommitColumn(_data); 86 //_data.FixSize(false); 87} 88 89void c4_FormatX::Define(int rows_, const t4_byte **ptr_) { 90 if (ptr_ != 0) 91 _data.PullLocation(*ptr_); 92 93 _data.SetRowCount(rows_); 94} 95 96void c4_FormatX::OldDefine(char, c4_Persist &pers_) { 97 pers_.FetchOldLocation(_data); 98 _data.SetRowCount(Owner().NumRows()); 99} 100 101void c4_FormatX::FlipBytes() { 102 _data.FlipBytes(); 103} 104 105int c4_FormatX::ItemSize(int index_) { 106 return _data.ItemSize(index_); 107} 108 109const void *c4_FormatX::Get(int index_, int &length_) { 110 return _data.Get(index_, length_); 111} 112 113void c4_FormatX::Set(int index_, const c4_Bytes &buf_) { 114 _data.Set(index_, buf_); 115} 116 117void c4_FormatX::Insert(int index_, const c4_Bytes &buf_, int count_) { 118 _data.Insert(index_, buf_, count_); 119} 120 121void c4_FormatX::Remove(int index_, int count_) { 122 _data.Remove(index_, count_); 123} 124 125void c4_FormatX::Unmapped() { 126 _data.ReleaseAllSegments(); 127} 128 129///////////////////////////////////////////////////////////////////////////// 130#if !q4_TINY 131///////////////////////////////////////////////////////////////////////////// 132 133class c4_FormatL: public c4_FormatX { 134 public: 135 c4_FormatL(const c4_Property &prop_, c4_HandlerSeq &seq_); 136 137 virtual void Define(int, const t4_byte **); 138 139 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 140}; 141 142///////////////////////////////////////////////////////////////////////////// 143 144c4_FormatL::c4_FormatL(const c4_Property &prop_, c4_HandlerSeq &seq_): 145 c4_FormatX(prop_, seq_, sizeof(t4_i64)) { 146 // force maximum size, autosizing more than 32 bits won't work 147 _data.SetAccessWidth(8 *sizeof(t4_i64)); 148} 149 150int c4_FormatL::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 151 d4_assert(b1_.Size() == sizeof(t4_i64)); 152 d4_assert(b2_.Size() == sizeof(t4_i64)); 153 154 t4_i64 v1 = *(const t4_i64*)b1_.Contents(); 155 t4_i64 v2 = *(const t4_i64*)b2_.Contents(); 156 157 return v1 == v2 ? 0 : v1 < v2 ? - 1: + 1; 158} 159 160void c4_FormatL::Define(int rows_, const t4_byte **ptr_) { 161 if (ptr_ == 0 && rows_ > 0) { 162 d4_assert(_data.ColSize() == 0); 163 _data.InsertData(0, rows_ *sizeof(t4_i64), true); 164 } 165 166 c4_FormatX::Define(rows_, ptr_); 167} 168 169///////////////////////////////////////////////////////////////////////////// 170 171class c4_FormatF: public c4_FormatX { 172 public: 173 c4_FormatF(const c4_Property &prop_, c4_HandlerSeq &seq_); 174 175 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 176}; 177 178///////////////////////////////////////////////////////////////////////////// 179 180c4_FormatF::c4_FormatF(const c4_Property &prop_, c4_HandlerSeq &seq_): 181 c4_FormatX(prop_, seq_, sizeof(float)){} 182 183int c4_FormatF::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 184 d4_assert(b1_.Size() == sizeof(float)); 185 d4_assert(b2_.Size() == sizeof(float)); 186 187 float v1 = *(const float*)b1_.Contents(); 188 float v2 = *(const float*)b2_.Contents(); 189 190 return v1 == v2 ? 0 : v1 < v2 ? - 1: + 1; 191} 192 193///////////////////////////////////////////////////////////////////////////// 194 195class c4_FormatD: public c4_FormatX { 196 public: 197 c4_FormatD(const c4_Property &prop_, c4_HandlerSeq &seq_); 198 199 virtual void Define(int, const t4_byte **); 200 201 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 202}; 203 204///////////////////////////////////////////////////////////////////////////// 205 206c4_FormatD::c4_FormatD(const c4_Property &prop_, c4_HandlerSeq &seq_): 207 c4_FormatX(prop_, seq_, sizeof(double)) { 208 // force maximum size, autosizing more than 32 bits won't work 209 _data.SetAccessWidth(8 *sizeof(double)); 210} 211 212int c4_FormatD::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 213 d4_assert(b1_.Size() == sizeof(double)); 214 d4_assert(b2_.Size() == sizeof(double)); 215 216 double v1 = *(const double*)b1_.Contents(); 217 double v2 = *(const double*)b2_.Contents(); 218 219 return v1 == v2 ? 0 : v1 < v2 ? - 1: + 1; 220} 221 222void c4_FormatD::Define(int rows_, const t4_byte **ptr_) { 223 if (ptr_ == 0 && rows_ > 0) { 224 d4_assert(_data.ColSize() == 0); 225 _data.InsertData(0, rows_ *sizeof(double), true); 226 } 227 228 c4_FormatX::Define(rows_, ptr_); 229} 230 231///////////////////////////////////////////////////////////////////////////// 232#endif // !q4_TINY 233///////////////////////////////////////////////////////////////////////////// 234 235/* 236Byte properties are used for raw bytes and for indirect (memo) data. 237 238There are two columns: the actual data and the item sizes. If the data 239is indirect, then the size is stored as a negative value. 240 241In addition, there is an in-memory-only vector of columns (_memos). 242Columns are created when asked for, and stay around until released with 243a commit call. If the column object exists and is not dirty, then it 244is either a real column (size < 0), or simply a duplicate of the data 245stored inline as bytes. 246 */ 247 248class c4_FormatB: public c4_FormatHandler { 249 public: 250 c4_FormatB(const c4_Property &prop_, c4_HandlerSeq &seq_); 251 virtual ~c4_FormatB(); 252 253 virtual void Define(int, const t4_byte **); 254 virtual void OldDefine(char type_, c4_Persist &); 255 virtual void Commit(c4_SaveContext &ar_); 256 257 virtual int ItemSize(int index_); 258 virtual const void *Get(int index_, int &length_); 259 virtual void Set(int index_, const c4_Bytes &buf_); 260 261 virtual void Insert(int index_, const c4_Bytes &buf_, int count_); 262 virtual void Remove(int index_, int count_); 263 264 virtual c4_Column *GetNthMemoCol(int index_, bool alloc_); 265 266 virtual void Unmapped(); 267 268 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 269 270 protected: 271 const void *GetOne(int index_, int &length_); 272 void SetOne(int index_, const c4_Bytes &buf_, bool ignoreMemos_ = false); 273 274 private: 275 t4_i32 Offset(int index_)const; 276 bool ShouldBeMemo(int length_)const; 277 int ItemLenOffCol(int index_, t4_i32 &off_, c4_Column * &col_); 278 bool CommitItem(c4_SaveContext &ar_, int index_); 279 void InitOffsets(c4_ColOfInts &sizes_); 280 281 c4_Column _data; 282 c4_ColOfInts _sizeCol; // 2001-11-27: keep, to track position on disk 283 c4_Column _memoCol; // 2001-11-27: keep, to track position on disk 284 c4_DWordArray _offsets; 285 c4_PtrArray _memos; 286 bool _recalc; // 2001-11-27: remember when to redo _{size,memo}Col 287}; 288 289///////////////////////////////////////////////////////////////////////////// 290 291c4_FormatB::c4_FormatB(const c4_Property &prop_, c4_HandlerSeq &seq_): 292 c4_FormatHandler(prop_, seq_), _data(seq_.Persist()), _sizeCol(seq_.Persist()) 293 , _memoCol(seq_.Persist()), _recalc(false) { 294 _offsets.SetSize(1, 100); 295 _offsets.SetAt(0, 0); 296} 297 298c4_FormatB::~c4_FormatB() { 299 // cleanup allocated columns 300 //better? for (int i = _memos.GetSize(); --i >= 0 ;) 301 for (int i = 0; i < _memos.GetSize(); ++i) 302 delete (c4_Column*)_memos.GetAt(i); 303} 304 305d4_inline t4_i32 c4_FormatB::Offset(int index_)const { 306 d4_assert((t4_i32)_offsets.GetAt(_offsets.GetSize() - 1) == _data.ColSize()); 307 d4_assert(_offsets.GetSize() == _memos.GetSize() + 1); 308 d4_assert(index_ < _offsets.GetSize()); 309 310 // extend offset vectors for missing empty entries at end 311 int n = _offsets.GetSize(); 312 d4_assert(n > 0); 313 314 if (index_ >= n) 315 index_ = n - 1; 316 317 return _offsets.GetAt(index_); 318} 319 320d4_inline bool c4_FormatB::ShouldBeMemo(int length_)const { 321 // items over 10000 bytes are always memos 322 // items up to 100 bytes are never memos 323 // 324 // else, memo only if the column would be under 1 Mb 325 // (assuming all items had the same size as this one) 326 // 327 // the effect is that as the number of rows increases, 328 // smaller and smaller items get turned into memos 329 // 330 // note that items which are no memo right now stay 331 // as is, and so do memos which have not been modified 332 333 int rows = _memos.GetSize() + 1; // avoids divide by zero 334 return length_ > 10000 || length_ > 100 && length_ > 1000000 / rows; 335} 336 337int c4_FormatB::ItemLenOffCol(int index_, t4_i32 &off_, c4_Column * &col_) { 338 col_ = (c4_Column*)_memos.GetAt(index_); 339 if (col_ != 0) { 340 off_ = 0; 341 return col_->ColSize(); 342 } 343 344 col_ = &_data; 345 off_ = Offset(index_); 346 return Offset(index_ + 1) - off_; 347} 348 349c4_Column *c4_FormatB::GetNthMemoCol(int index_, bool alloc_) { 350 t4_i32 start; 351 c4_Column *col; 352 int n = ItemLenOffCol(index_, start, col); 353 354 if (col == &_data && alloc_) { 355 col = d4_new c4_Column(_data.Persist()); 356 _memos.SetAt(index_, col); 357 358 if (n > 0) 359 if (_data.IsDirty()) { 360 c4_Bytes temp; 361 _data.FetchBytes(start, n, temp, true); 362 col->SetBuffer(n); 363 col->StoreBytes(0, temp); 364 } else 365 col->SetLocation(_data.Position() + start, n); 366 } 367 368 return col; 369} 370 371void c4_FormatB::Unmapped() { 372 _data.ReleaseAllSegments(); 373 _sizeCol.ReleaseAllSegments(); 374 _memoCol.ReleaseAllSegments(); 375 376 for (int i = 0; i < _memos.GetSize(); ++i) { 377 c4_Column *cp = (c4_Column*)_memos.GetAt(i); 378 if (cp != 0) 379 cp->ReleaseAllSegments(); 380 } 381} 382 383void c4_FormatB::Define(int, const t4_byte **ptr_) { 384 d4_assert(_memos.GetSize() == 0); 385 386 if (ptr_ != 0) { 387 _data.PullLocation(*ptr_); 388 if (_data.ColSize() > 0) 389 _sizeCol.PullLocation(*ptr_); 390 _memoCol.PullLocation(*ptr_); 391 } 392 393 // everything below this point could be delayed until use 394 // in that case, watch out that column space use is properly tracked 395 396 InitOffsets(_sizeCol); 397 398 if (_memoCol.ColSize() > 0) { 399 c4_Bytes walk; 400 _memoCol.FetchBytes(0, _memoCol.ColSize(), walk, true); 401 402 const t4_byte *p = walk.Contents(); 403 404 for (int row = 0; p < walk.Contents() + walk.Size(); ++row) { 405 row += c4_Column::PullValue(p); 406 d4_assert(row < _memos.GetSize()); 407 408 c4_Column *mc = d4_new c4_Column(_data.Persist()); 409 d4_assert(mc != 0); 410 _memos.SetAt(row, mc); 411 412 mc->PullLocation(p); 413 } 414 415 d4_assert(p == walk.Contents() + walk.Size()); 416 } 417} 418 419void c4_FormatB::OldDefine(char type_, c4_Persist &pers_) { 420 int rows = Owner().NumRows(); 421 422 c4_ColOfInts sizes(_data.Persist()); 423 424 if (type_ == 'M') { 425 InitOffsets(sizes); 426 427 c4_ColOfInts szVec(_data.Persist()); 428 pers_.FetchOldLocation(szVec); 429 szVec.SetRowCount(rows); 430 431 c4_ColOfInts posVec(_data.Persist()); 432 pers_.FetchOldLocation(posVec); 433 posVec.SetRowCount(rows); 434 435 for (int r = 0; r < rows; ++r) { 436 t4_i32 sz = szVec.GetInt(r); 437 if (sz > 0) { 438 c4_Column *mc = d4_new c4_Column(_data.Persist()); 439 d4_assert(mc != 0); 440 _memos.SetAt(r, mc); 441 442 mc->SetLocation(posVec.GetInt(r), sz); 443 } 444 } 445 } else { 446 pers_.FetchOldLocation(_data); 447 448 if (type_ == 'B') { 449 pers_.FetchOldLocation(sizes); 450 451#if !q4_OLD_IS_ALWAYS_V2 452 453 // WARNING - HUGE HACK AHEAD - THIS IS NOT 100% FULLPROOF! 454 // 455 // The above is correct for MK versions 2.0 and up, but *NOT* 456 // for MK 1.8.6 datafiles, which store sizes first (OUCH!!!). 457 // This means that there is not a 100% safe way to auto-convert 458 // both 1.8.6 and 2.0 files - since there is no way to detect 459 // unambiguously which version a datafile is. All we can do, 460 // is to carefully check both vectors, and *hope* that only one 461 // of them is valid as sizes vector. This problem applies to 462 // the 'B' (bytes) property type only, and only pre 2.0 files. 463 // 464 // To build a version which *always* converts assuming 1.8.6, 465 // add flag "-Dq4_OLD_IS_PRE_V2" to the compiler command line. 466 // Conversely, "-Dq4_OLD_IS_ALWAYS_V2" forces 2.0 conversion. 467 468 if (rows > 0) { 469 t4_i32 s1 = sizes.ColSize(); 470 t4_i32 s2 = _data.ColSize(); 471 472#if !q4_OLD_IS_PRE_V2 473 // if the size vector is clearly impossible, swap vectors 474 bool fix = c4_ColOfInts::CalcAccessWidth(rows, s1) < 0; 475 476 // if the other vector might be valid as well, check further 477 if (!fix && c4_ColOfInts::CalcAccessWidth(rows, s2) >= 0) { 478 sizes.SetRowCount(rows); 479 t4_i32 total = 0; 480 for (int i = 0; i < rows; ++i) { 481 t4_i32 w = sizes.GetInt(i); 482 if (w < 0 || total > s2) { 483 total = - 1; 484 break; 485 } 486 total += w; 487 } 488 489 // if the sizes don't add up, swap vectors 490 fix = total != s2; 491 } 492 493 if (fix) 494#endif 495 { 496 t4_i32 p1 = sizes.Position(); 497 t4_i32 p2 = _data.Position(); 498 _data.SetLocation(p1, s1); 499 sizes.SetLocation(p2, s2); 500 } 501 } 502#endif 503 InitOffsets(sizes); 504 } else { 505 d4_assert(type_ == 'S'); 506 507 sizes.SetRowCount(rows); 508 509 t4_i32 pos = 0; 510 t4_i32 lastEnd = 0; 511 int k = 0; 512 513 c4_ColIter iter(_data, 0, _data.ColSize()); 514 while (iter.Next()) { 515 const t4_byte *p = iter.BufLoad(); 516 for (int j = 0; j < iter.BufLen(); ++j) 517 if (!p[j]) { 518 sizes.SetInt(k++, pos + j + 1-lastEnd); 519 lastEnd = pos + j + 1; 520 } 521 522 pos += iter.BufLen(); 523 } 524 525 d4_assert(pos == _data.ColSize()); 526 527 if (lastEnd < pos) { 528 // last entry had no zero byte 529 _data.InsertData(pos++, 1, true); 530 sizes.SetInt(k, pos - lastEnd); 531 } 532 533 InitOffsets(sizes); 534 535 // get rid of entries with just a null byte 536 for (int r = 0; r < rows; ++r) 537 if (c4_FormatB::ItemSize(r) == 1) 538 SetOne(r, c4_Bytes()); 539 } 540 } 541} 542 543void c4_FormatB::InitOffsets(c4_ColOfInts &sizes_) { 544 int rows = Owner().NumRows(); 545 546 if (sizes_.RowCount() != rows) { 547 sizes_.SetRowCount(rows); 548 } 549 550 _memos.SetSize(rows); 551 _offsets.SetSize(rows + 1); 552 553 if (_data.ColSize() > 0) { 554 t4_i32 total = 0; 555 556 for (int r = 0; r < rows; ++r) { 557 int n = sizes_.GetInt(r); 558 d4_assert(n >= 0); 559 total += n; 560 _offsets.SetAt(r + 1, total); 561 } 562 563 d4_assert(total == _data.ColSize()); 564 } 565 566} 567 568int c4_FormatB::ItemSize(int index_) { 569 t4_i32 start; 570 c4_Column *col; 571 return ItemLenOffCol(index_, start, col); 572} 573 574const void *c4_FormatB::GetOne(int index_, int &length_) { 575 t4_i32 start; 576 c4_Column *cp; 577 length_ = ItemLenOffCol(index_, start, cp); 578 d4_assert(length_ >= 0); 579 580 if (length_ == 0) 581 return ""; 582 583 return cp->FetchBytes(start, length_, Owner().Buffer(), false); 584} 585 586const void *c4_FormatB::Get(int index_, int &length_) { 587 return GetOne(index_, length_); 588} 589 590void c4_FormatB::SetOne(int index_, const c4_Bytes &xbuf_, bool ignoreMemos_) { 591 // this fixes bug in 2.4.0 when copying string from higher row 592 // TODO: this fix is very conservative, figure out when to copy 593 // (can probably look at pointer to see whether it's from us) 594 int sz = xbuf_.Size(); 595 c4_Bytes buf_(xbuf_.Contents(), sz, 0 < sz && sz <= c4_Column::kSegMax); 596 597 c4_Column *cp = &_data; 598 t4_i32 start = Offset(index_); 599 int len = Offset(index_ + 1) - start; 600 601 if (!ignoreMemos_ && _memos.GetAt(index_) != 0) 602 len = ItemLenOffCol(index_, start, cp); 603 604 int m = buf_.Size(); 605 int n = m - len; 606 607 if (n > 0) 608 cp->Grow(start, n); 609 else if (n < 0) 610 cp->Shrink(start, - n); 611 else if (m == 0) 612 return ; 613 // no size change and no contents 614 615 _recalc = true; 616 617 cp->StoreBytes(start, buf_); 618 619 if (n && cp == &_data) { 620 // if size has changed 621 int k = _offsets.GetSize() - 1; 622 623 // if filling in an empty entry at end: extend offsets first 624 if (m > 0 && index_ >= k) { 625 _offsets.InsertAt(k, _offsets.GetAt(k), index_ - k + 1); 626 627 k = index_ + 1; 628 d4_assert(k == _offsets.GetSize() - 1); 629 } 630 631 // adjust following entry offsets 632 while (++index_ <= k) 633 _offsets.ElementAt(index_) += n; 634 } 635 636 d4_assert((t4_i32)_offsets.GetAt(_offsets.GetSize() - 1) == _data.ColSize()); 637} 638 639void c4_FormatB::Set(int index_, const c4_Bytes &buf_) { 640 SetOne(index_, buf_); 641} 642 643int c4_FormatB::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 644 int n = b1_.Size(); 645 if (n > b2_.Size()) 646 n = b2_.Size(); 647 648 int f = memcmp(b1_.Contents(), b2_.Contents(), n); 649 return f ? f : b1_.Size() - b2_.Size(); 650} 651 652void c4_FormatB::Insert(int index_, const c4_Bytes &buf_, int count_) { 653 d4_assert(count_ > 0); 654 655 _recalc = true; 656 657 int m = buf_.Size(); 658 t4_i32 off = Offset(index_); 659 660 _memos.InsertAt(index_, 0, count_); 661 662 // insert the appropriate number of bytes 663 t4_i32 n = count_ *(t4_i32)m; 664 if (n > 0) { 665 _data.Grow(off, n); 666 667 // store as many copies as needed, but may have to do it in chunks 668 int spos = 0; 669 670 c4_ColIter iter(_data, off, off + n); 671 while (iter.Next(m - spos)) { 672 memcpy(iter.BufSave(), buf_.Contents() + spos, iter.BufLen()); 673 674 spos += iter.BufLen(); 675 if (spos >= m) 676 spos = 0; 677 } 678 679 d4_assert(spos == 0); // must have copied an exact multiple of the data 680 } 681 682 // define offsets of the new entries 683 _offsets.InsertAt(index_, 0, count_); 684 d4_assert(_offsets.GetSize() <= _memos.GetSize() + 1); 685 686 while (--count_ >= 0) { 687 _offsets.SetAt(index_++, off); 688 off += m; 689 } 690 691 d4_assert(index_ < _offsets.GetSize()); 692 693 // adjust all following entries 694 while (index_ < _offsets.GetSize()) 695 _offsets.ElementAt(index_++) += n; 696 697 d4_assert((t4_i32)_offsets.GetAt(index_ - 1) == _data.ColSize()); 698 d4_assert(index_ <= _memos.GetSize() + 1); 699} 700 701void c4_FormatB::Remove(int index_, int count_) { 702 _recalc = true; 703 704 t4_i32 off = Offset(index_); 705 t4_i32 n = Offset(index_ + count_) - off; 706 d4_assert(n >= 0); 707 708 // remove the columns, if present 709 for (int i = 0; i < count_; ++i) 710 delete (c4_Column*)_memos.GetAt(index_ + i); 711 _memos.RemoveAt(index_, count_); 712 713 if (n > 0) 714 _data.Shrink(off, n); 715 716 _offsets.RemoveAt(index_, count_); 717 718 d4_assert(index_ < _offsets.GetSize()); 719 720 // adjust all following entries 721 while (index_ < _offsets.GetSize()) 722 _offsets.ElementAt(index_++) -= n; 723 724 d4_assert((t4_i32)_offsets.GetAt(index_ - 1) == _data.ColSize()); 725 d4_assert(index_ <= _memos.GetSize() + 1); 726} 727 728void c4_FormatB::Commit(c4_SaveContext &ar_) { 729 int rows = _memos.GetSize(); 730 d4_assert(rows > 0); 731 732 bool full = _recalc || ar_.Serializing(); 733 734 if (!full) 735 for (int i = 0; i < rows; ++i) { 736 c4_Column *col = (c4_Column*)_memos.GetAt(i); 737 if (col != 0) { 738 full = true; 739 break; 740 } 741 } 742 d4_assert(_recalc || _sizeCol.RowCount() == rows); 743 744 if (full) { 745 _memoCol.SetBuffer(0); 746 _sizeCol.SetBuffer(0); 747 _sizeCol.SetAccessWidth(0); 748 _sizeCol.SetRowCount(rows); 749 750 int skip = 0; 751 752 c4_Column *saved = ar_.SetWalkBuffer(&_memoCol); 753 754 for (int r = 0; r < rows; ++r) { 755 ++skip; 756 757 t4_i32 start; 758 c4_Column *col; 759 int len = ItemLenOffCol(r, start, col); 760 761 bool oldMemo = col != &_data; 762 bool newMemo = ShouldBeMemo(len); 763 764 if (!oldMemo && newMemo) { 765 col = GetNthMemoCol(r, true); 766 d4_assert(col != &_data); 767 //? start = 0; 768 } 769 770 c4_Bytes temp; 771 772 if (newMemo) { 773 // it now is a memo, inlined data will be empty 774 ar_.StoreValue(skip - 1); 775 skip = 0; 776 ar_.CommitColumn(*col); 777 } else if (!oldMemo) { 778 // it was no memo, done if it hasn't become one 779 _sizeCol.SetInt(r, len); 780 continue; 781 } else { 782 // it was a memo, but it no longer is 783 d4_assert(start == 0); 784 if (len > 0) { 785 _sizeCol.SetInt(r, len); 786 col->FetchBytes(start, len, temp, true); 787 delete (c4_Column*)_memos.GetAt(r); // 28-11-2001: fix mem leak 788 _memos.SetAt(r, 0); // 02-11-2001: fix for use after commit 789 } 790 } 791 792 SetOne(r, temp, true); // bypass current memo pointer 793 } 794 795 ar_.SetWalkBuffer(saved); 796 } 797 798 ar_.CommitColumn(_data); 799 800 if (_data.ColSize() > 0) { 801 _sizeCol.FixSize(true); 802 ar_.CommitColumn(_sizeCol); 803 //_sizeCol.FixSize(false); 804 } 805 806 ar_.CommitColumn(_memoCol); 807 808 // need a way to find out when the data has been committed (on 2nd pass) 809 // both _sizeCol and _memoCol will be clean again when it has 810 // but be careful because dirty flag is only useful if size is nonzero 811 if (_recalc && !ar_.Serializing()) 812 _recalc = _sizeCol.ColSize() > 0 && _sizeCol.IsDirty() || _memoCol.ColSize() 813 > 0 && _memoCol.IsDirty(); 814} 815 816///////////////////////////////////////////////////////////////////////////// 817 818class c4_FormatS: public c4_FormatB { 819 public: 820 c4_FormatS(const c4_Property &prop_, c4_HandlerSeq &seq_); 821 822 virtual int ItemSize(int index_); 823 virtual const void *Get(int index_, int &length_); 824 virtual void Set(int index_, const c4_Bytes &buf_); 825 826 virtual void Insert(int index_, const c4_Bytes &buf_, int count_); 827 828 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 829}; 830 831///////////////////////////////////////////////////////////////////////////// 832 833c4_FormatS::c4_FormatS(const c4_Property &prop_, c4_HandlerSeq &seq_): 834 c4_FormatB(prop_, seq_){} 835 836int c4_FormatS::ItemSize(int index_) { 837 int n = c4_FormatB::ItemSize(index_) - 1; 838 return n >= 0 ? n : 0; 839} 840 841const void *c4_FormatS::Get(int index_, int &length_) { 842 const void *ptr = GetOne(index_, length_); 843 844 if (length_ == 0) { 845 length_ = 1; 846 ptr = ""; 847 } 848 849 d4_assert(((const char*)ptr)[length_ - 1] == 0); 850 return ptr; 851} 852 853void c4_FormatS::Set(int index_, const c4_Bytes &buf_) { 854 int m = buf_.Size(); 855 if (--m >= 0) { 856 d4_assert(buf_.Contents()[m] == 0); 857 if (m == 0) { 858 SetOne(index_, c4_Bytes()); // don't store data for empty strings 859 return ; 860 } 861 } 862 863 SetOne(index_, buf_); 864} 865 866int c4_FormatS::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 867 c4_String v1((const char*)b1_.Contents(), b1_.Size()); 868 c4_String v2((const char*)b2_.Contents(), b2_.Size()); 869 870 return v1.CompareNoCase(v2); 871} 872 873void c4_FormatS::Insert(int index_, const c4_Bytes &buf_, int count_) { 874 d4_assert(count_ > 0); 875 876 int m = buf_.Size(); 877 if (--m >= 0) { 878 d4_assert(buf_.Contents()[m] == 0); 879 if (m == 0) { 880 c4_FormatB::Insert(index_, c4_Bytes(), count_); 881 return ; 882 } 883 } 884 885 c4_FormatB::Insert(index_, buf_, count_); 886} 887 888///////////////////////////////////////////////////////////////////////////// 889 890class c4_FormatV: public c4_FormatHandler { 891 public: 892 c4_FormatV(const c4_Property &prop_, c4_HandlerSeq &seq_); 893 virtual ~c4_FormatV(); 894 895 virtual void Define(int rows_, const t4_byte **ptr_); 896 virtual void OldDefine(char type_, c4_Persist &); 897 virtual void Commit(c4_SaveContext &ar_); 898 899 virtual void FlipBytes(); 900 901 virtual int ItemSize(int index_); 902 virtual const void *Get(int index_, int &length_); 903 virtual void Set(int index_, const c4_Bytes &buf_); 904 905 virtual void Insert(int index_, const c4_Bytes &buf_, int count_); 906 virtual void Remove(int index_, int count_); 907 908 virtual void Unmapped(); 909 virtual bool HasSubview(int index_); 910 911 static int DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_); 912 913 private: 914 c4_HandlerSeq &At(int index_); 915 void Replace(int index_, c4_HandlerSeq *seq_); 916 void SetupAllSubviews(); 917 void ForgetSubview(int index_); 918 919 c4_Column _data; 920 c4_PtrArray _subSeqs; 921 bool _inited; 922}; 923 924///////////////////////////////////////////////////////////////////////////// 925 926c4_FormatV::c4_FormatV(const c4_Property &prop_, c4_HandlerSeq &seq_): 927 c4_FormatHandler(prop_, seq_), _data(seq_.Persist()), _inited(false){} 928 929c4_FormatV::~c4_FormatV() { 930 for (int i = 0; i < _subSeqs.GetSize(); ++i) 931 ForgetSubview(i); 932} 933 934c4_HandlerSeq &c4_FormatV::At(int index_) { 935 d4_assert(_inited); 936 937 c4_HandlerSeq * &hs = (c4_HandlerSeq * &)_subSeqs.ElementAt(index_); 938 if (hs == 0) { 939 hs = d4_new c4_HandlerSeq(Owner(), this); 940 hs->IncRef(); 941 } 942 943 return *hs; 944} 945 946void c4_FormatV::SetupAllSubviews() { 947 d4_assert(!_inited); 948 _inited = true; 949 950 if (_data.ColSize() > 0) { 951 c4_Bytes temp; 952 _data.FetchBytes(0, _data.ColSize(), temp, true); 953 const t4_byte *ptr = temp.Contents(); 954 955 for (int r = 0; r < _subSeqs.GetSize(); ++r) { 956 // don't materialize subview if it is empty 957 // duplicates code which is in c4_HandlerSeq::Prepare 958 const t4_byte *p2 = ptr; 959 d4_dbgdef(t4_i32 sias = )c4_Column::PullValue(p2); 960 d4_assert(sias == 0); // not yet 961 962 if (c4_Column::PullValue(p2) > 0) 963 At(r).Prepare(&ptr, false); 964 else 965 ptr = p2; 966 } 967 968 d4_assert(ptr == temp.Contents() + temp.Size()); 969 } 970} 971 972void c4_FormatV::Define(int rows_, const t4_byte **ptr_) { 973 if (_inited) { 974 // big oops: a root handler already contains data 975 976 for (int i = 0; i < _subSeqs.GetSize(); ++i) 977 ForgetSubview(i); 978 979 _inited = false; 980 } 981 982 _subSeqs.SetSize(rows_); 983 if (ptr_ != 0) 984 _data.PullLocation(*ptr_); 985} 986 987void c4_FormatV::OldDefine(char, c4_Persist &pers_) { 988 int rows = Owner().NumRows(); 989 _subSeqs.SetSize(rows); 990 991 for (int i = 0; i < rows; ++i) { 992 int n = pers_.FetchOldValue(); 993 if (n) { 994 // 14-11-2000: do not create again (this causes a mem leak) 995 // 04-12-2000: but do create if absent (fixes occasional crash) 996 c4_HandlerSeq *hs = (c4_HandlerSeq*)_subSeqs.GetAt(i); 997 if (hs == 0) { 998 hs = d4_new c4_HandlerSeq(Owner(), this); 999 _subSeqs.SetAt(i, hs); 1000 hs->IncRef(); 1001 } 1002 hs->SetNumRows(n); 1003 hs->OldPrepare(); 1004 } 1005 } 1006} 1007 1008void c4_FormatV::FlipBytes() { 1009 if (!_inited) 1010 SetupAllSubviews(); 1011 1012 for (int i = 0; i < _subSeqs.GetSize(); ++i) 1013 At(i).FlipAllBytes(); 1014} 1015 1016int c4_FormatV::ItemSize(int index_) { 1017 if (!_inited) 1018 SetupAllSubviews(); 1019 1020 // 06-02-2002: avoid creating empty subview 1021 c4_HandlerSeq *hs = (c4_HandlerSeq * &)_subSeqs.ElementAt(index_); 1022 return hs == 0 ? 0 : hs->NumRows(); 1023} 1024 1025const void *c4_FormatV::Get(int index_, int &length_) { 1026 if (!_inited) 1027 SetupAllSubviews(); 1028 1029 At(index_); // forces existence of a real entry 1030 c4_HandlerSeq * &e = (c4_HandlerSeq * &)_subSeqs.ElementAt(index_); 1031 1032 length_ = sizeof(c4_HandlerSeq **); 1033 return &e; 1034} 1035 1036void c4_FormatV::Set(int index_, const c4_Bytes &buf_) { 1037 d4_assert(buf_.Size() == sizeof(c4_Sequence*)); 1038 1039 if (!_inited) 1040 SetupAllSubviews(); 1041 1042 c4_HandlerSeq *value = *(c4_HandlerSeq *const*)buf_.Contents(); 1043 1044 if (value != &At(index_)) 1045 Replace(index_, value); 1046} 1047 1048void c4_FormatV::Replace(int index_, c4_HandlerSeq *seq_) { 1049 if (!_inited) 1050 SetupAllSubviews(); 1051 1052 c4_HandlerSeq * &curr = (c4_HandlerSeq * &)_subSeqs.ElementAt(index_); 1053 if (seq_ == curr) 1054 return ; 1055 1056 if (curr != 0) { 1057 d4_assert(&curr->Parent() == &Owner()); 1058 curr->DetachFromParent(); 1059 curr->DetachFromStorage(true); 1060 1061 curr->DecRef(); 1062 curr = 0; 1063 } 1064 1065 if (seq_) { 1066 int n = seq_->NumRows(); 1067 1068 c4_HandlerSeq &t = At(index_); 1069 d4_assert(t.NumRows() == 0); 1070 1071 t.Resize(n); 1072 1073 c4_Bytes data; 1074 1075 // this dest seq has only the persistent handlers 1076 // and maybe in a different order 1077 // create any others we need as temporary properties 1078 for (int i = 0; i < seq_->NumHandlers(); ++i) { 1079 c4_Handler &h1 = seq_->NthHandler(i); 1080 1081 int j = t.PropIndex(h1.Property()); 1082 d4_assert(j >= 0); 1083 1084 c4_Handler &h2 = t.NthHandler(j); 1085 1086 for (int k = 0; k < n; ++k) 1087 if (seq_->Get(k, h1.PropId(), data)) 1088 h2.Set(k, data); 1089 } 1090 } 1091} 1092 1093int c4_FormatV::DoCompare(const c4_Bytes &b1_, const c4_Bytes &b2_) { 1094 d4_assert(b1_.Size() == sizeof(c4_Sequence*)); 1095 d4_assert(b2_.Size() == sizeof(c4_Sequence*)); 1096 1097 c4_View v1 = *(c4_Sequence *const*)b1_.Contents(); 1098 c4_View v2 = *(c4_Sequence *const*)b2_.Contents(); 1099 1100 return v1.Compare(v2); 1101} 1102 1103void c4_FormatV::Insert(int index_, const c4_Bytes &buf_, int count_) { 1104 d4_assert(buf_.Size() == sizeof(c4_Sequence*)); 1105 d4_assert(count_ > 0); 1106 1107 // can only insert an empty entry! 1108 d4_assert(*(c4_Sequence *const*)buf_.Contents() == 0); 1109 1110 if (!_inited) 1111 SetupAllSubviews(); 1112 1113 _subSeqs.InsertAt(index_, 0, count_); 1114 _data.SetBuffer(0); // 2004-01-18 force dirty 1115} 1116 1117void c4_FormatV::Remove(int index_, int count_) { 1118 d4_assert(count_ > 0); 1119 1120 if (!_inited) 1121 SetupAllSubviews(); 1122 1123 for (int i = 0; i < count_; ++i) 1124 ForgetSubview(index_ + i); 1125 1126 _subSeqs.RemoveAt(index_, count_); 1127 _data.SetBuffer(0); // 2004-01-18 force dirty 1128} 1129 1130void c4_FormatV::Unmapped() { 1131 if (_inited) 1132 for (int i = 0; i < _subSeqs.GetSize(); ++i) 1133 if (HasSubview(i)) { 1134 c4_HandlerSeq &hs = At(i); 1135 hs.UnmappedAll(); 1136 if (hs.NumRefs() == 1 && hs.NumRows() == 0) 1137 ForgetSubview(i); 1138 } 1139 1140 _data.ReleaseAllSegments(); 1141} 1142 1143bool c4_FormatV::HasSubview(int index_) { 1144 if (!_inited) 1145 SetupAllSubviews(); 1146 1147 return _subSeqs.ElementAt(index_) != 0; 1148} 1149 1150void c4_FormatV::ForgetSubview(int index_) { 1151 c4_HandlerSeq * &seq = (c4_HandlerSeq * &)_subSeqs.ElementAt(index_); 1152 if (seq != 0) { 1153 d4_assert(&seq->Parent() == &Owner()); 1154 seq->DetachFromParent(); 1155 seq->DetachFromStorage(true); 1156 seq->UnmappedAll(); 1157 seq->DecRef(); 1158 seq = 0; 1159 } 1160} 1161 1162void c4_FormatV::Commit(c4_SaveContext &ar_) { 1163 if (!_inited) 1164 SetupAllSubviews(); 1165 1166 int rows = _subSeqs.GetSize(); 1167 d4_assert(rows > 0); 1168 1169 c4_Column temp(0); 1170 c4_Column *saved = ar_.SetWalkBuffer(&temp); 1171 1172 for (int r = 0; r < rows; ++r) 1173 if (HasSubview(r)) { 1174 c4_HandlerSeq &hs = At(r); 1175 ar_.CommitSequence(hs, false); 1176 if (hs.NumRefs() == 1 && hs.NumRows() == 0) 1177 ForgetSubview(r); 1178 } else { 1179 ar_.StoreValue(0); // sias 1180 ar_.StoreValue(0); // row count 1181 } 1182 1183 ar_.SetWalkBuffer(saved); 1184 1185 c4_Bytes buf; 1186 temp.FetchBytes(0, temp.ColSize(), buf, true); 1187 1188 bool changed = temp.ColSize() != _data.ColSize(); 1189 1190 if (!changed) { 1191 c4_Bytes buf2; 1192 _data.FetchBytes(0, _data.ColSize(), buf2, true); 1193 changed = buf != buf2; 1194 } 1195 1196 if (changed) { 1197 _data.SetBuffer(buf.Size()); 1198 _data.StoreBytes(0, buf); 1199 } 1200 1201 ar_.CommitColumn(_data); 1202} 1203 1204///////////////////////////////////////////////////////////////////////////// 1205 1206c4_Handler *f4_CreateFormat(const c4_Property &prop_, c4_HandlerSeq &seq_) { 1207 switch (prop_.Type()) { 1208 case 'I': 1209 return d4_new c4_FormatX(prop_, seq_); 1210#if !q4_TINY 1211 case 'L': 1212 return d4_new c4_FormatL(prop_, seq_); 1213 case 'F': 1214 return d4_new c4_FormatF(prop_, seq_); 1215 case 'D': 1216 return d4_new c4_FormatD(prop_, seq_); 1217#endif 1218 case 'B': 1219 return d4_new c4_FormatB(prop_, seq_); 1220 case 'S': 1221 return d4_new c4_FormatS(prop_, seq_); 1222 case 'V': 1223 return d4_new c4_FormatV(prop_, seq_); 1224 } 1225 1226 d4_assert(0); 1227 // 2004-01-16 turn bad definition type into an int property to avoid crash 1228 return d4_new c4_FormatX(c4_IntProp(prop_.Name()), seq_); 1229} 1230 1231///////////////////////////////////////////////////////////////////////////// 1232 1233int f4_ClearFormat(char type_) { 1234 switch (type_) { 1235 case 'I': 1236 return sizeof(t4_i32); 1237#if !q4_TINY 1238 case 'L': 1239 return sizeof(t4_i64); 1240 case 'F': 1241 return sizeof(float); 1242 case 'D': 1243 return sizeof(double); 1244#endif 1245 case 'S': 1246 return 1; 1247 case 'V': 1248 return sizeof(c4_Sequence*); 1249 } 1250 1251 return 0; 1252} 1253 1254///////////////////////////////////////////////////////////////////////////// 1255 1256int f4_CompareFormat(char type_, const c4_Bytes &b1_, const c4_Bytes &b2_) { 1257 switch (type_) { 1258 case 'I': 1259 return c4_FormatX::DoCompare(b1_, b2_); 1260#if !q4_TINY 1261 case 'L': 1262 return c4_FormatL::DoCompare(b1_, b2_); 1263 case 'F': 1264 return c4_FormatF::DoCompare(b1_, b2_); 1265 case 'D': 1266 return c4_FormatD::DoCompare(b1_, b2_); 1267#endif 1268 case 'B': 1269 return c4_FormatB::DoCompare(b1_, b2_); 1270 case 'S': 1271 return c4_FormatS::DoCompare(b1_, b2_); 1272 case 'V': 1273 return c4_FormatV::DoCompare(b1_, b2_); 1274 } 1275 1276 d4_assert(0); 1277 return 0; 1278} 1279 1280///////////////////////////////////////////////////////////////////////////// 1281