1176998Sphk/*- 2176998Sphk * Copyright (c) 2005-2008 Poul-Henning Kamp 3176998Sphk * All rights reserved. 4176998Sphk * 5176998Sphk * Redistribution and use in source and binary forms, with or without 6176998Sphk * modification, are permitted provided that the following conditions 7176998Sphk * are met: 8176998Sphk * 1. Redistributions of source code must retain the above copyright 9176998Sphk * notice, this list of conditions and the following disclaimer. 10176998Sphk * 2. Redistributions in binary form must reproduce the above copyright 11176998Sphk * notice, this list of conditions and the following disclaimer in the 12176998Sphk * documentation and/or other materials provided with the distribution. 13176998Sphk * 14176998Sphk * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15176998Sphk * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16176998Sphk * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17176998Sphk * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18176998Sphk * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19176998Sphk * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20176998Sphk * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21176998Sphk * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22176998Sphk * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23176998Sphk * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24176998Sphk * SUCH DAMAGE. 25176998Sphk * 26176998Sphk * $FreeBSD$ 27176998Sphk */ 28176998Sphk 29176998Sphk#include <assert.h> 30176998Sphk#include <stdio.h> 31176998Sphk#include <string.h> 32176998Sphk#include <stdlib.h> 33176998Sphk#include <unistd.h> 34219095Sphk#include <stdint.h> 35176998Sphk#include <time.h> 36176998Sphk#include <sys/endian.h> 37176998Sphk 38176998Sphk#include <zlib.h> 39176998Sphk 40176998Sphk#include "fifolog.h" 41176998Sphk#include "libfifolog_int.h" 42176998Sphk#include "fifolog_write.h" 43176998Sphk#include "miniobj.h" 44176998Sphk 45219095Sphkstatic int fifolog_write_gzip(struct fifolog_writer *f, time_t now); 46219095Sphk 47176998Sphk#define ALLOC(ptr, size) do { \ 48176998Sphk (*(ptr)) = calloc(size, 1); \ 49176998Sphk assert(*(ptr) != NULL); \ 50176998Sphk} while (0) 51176998Sphk 52176998Sphk 53176998Sphkconst char *fifolog_write_statnames[] = { 54219027Sphk [FIFOLOG_PT_BYTES_PRE] = "Bytes before compression", 55219027Sphk [FIFOLOG_PT_BYTES_POST] = "Bytes after compression", 56219027Sphk [FIFOLOG_PT_WRITES] = "Writes", 57219027Sphk [FIFOLOG_PT_FLUSH] = "Flushes", 58219027Sphk [FIFOLOG_PT_SYNC] = "Syncs", 59219027Sphk [FIFOLOG_PT_RUNTIME] = "Runtime" 60176998Sphk}; 61176998Sphk 62219095Sphk/********************************************************************** 63176998Sphk * Check that everything is all right 64176998Sphk */ 65176998Sphkstatic void 66176998Sphkfifolog_write_assert(const struct fifolog_writer *f) 67176998Sphk{ 68176998Sphk 69176998Sphk CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); 70176998Sphk assert(f->ff->zs->next_out + f->ff->zs->avail_out == \ 71216257Sphk f->obuf + f->obufsize); 72176998Sphk} 73176998Sphk 74219095Sphk/********************************************************************** 75219095Sphk * Allocate/Destroy a new fifolog writer instance 76219095Sphk */ 77219095Sphk 78176998Sphkstruct fifolog_writer * 79176998Sphkfifolog_write_new(void) 80176998Sphk{ 81176998Sphk struct fifolog_writer *f; 82176998Sphk 83216257Sphk ALLOC_OBJ(f, FIFOLOG_WRITER_MAGIC); 84216257Sphk assert(f != NULL); 85176998Sphk return (f); 86176998Sphk} 87176998Sphk 88176998Sphkvoid 89176998Sphkfifolog_write_destroy(struct fifolog_writer *f) 90176998Sphk{ 91219095Sphk 92219095Sphk free(f->obuf); 93219095Sphk free(f->ibuf); 94219095Sphk FREE_OBJ(f); 95176998Sphk} 96176998Sphk 97219095Sphk/********************************************************************** 98219095Sphk * Open/Close the fifolog 99219095Sphk */ 100219095Sphk 101176998Sphkvoid 102176998Sphkfifolog_write_close(struct fifolog_writer *f) 103176998Sphk{ 104219095Sphk time_t now; 105176998Sphk 106176998Sphk CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); 107219095Sphk fifolog_write_assert(f); 108219095Sphk 109219095Sphk f->cleanup = 1; 110219095Sphk time(&now); 111219095Sphk fifolog_write_gzip(f, now); 112219095Sphk fifolog_write_assert(f); 113176998Sphk fifolog_int_close(&f->ff); 114177381Sphk free(f->ff); 115176998Sphk} 116176998Sphk 117176998Sphkconst char * 118219095Sphkfifolog_write_open(struct fifolog_writer *f, const char *fn, 119219095Sphk unsigned writerate, unsigned syncrate, unsigned compression) 120176998Sphk{ 121176998Sphk const char *es; 122176998Sphk int i; 123176998Sphk time_t now; 124176998Sphk off_t o; 125176998Sphk 126176998Sphk CHECK_OBJ_NOTNULL(f, FIFOLOG_WRITER_MAGIC); 127176998Sphk 128176998Sphk /* Check for legal compression value */ 129219095Sphk if (compression > Z_BEST_COMPRESSION) 130176998Sphk return ("Illegal compression value"); 131176998Sphk 132176998Sphk f->writerate = writerate; 133176998Sphk f->syncrate = syncrate; 134176998Sphk f->compression = compression; 135176998Sphk 136176998Sphk /* Reset statistics */ 137176998Sphk memset(f->cnt, 0, sizeof f->cnt); 138176998Sphk 139176998Sphk es = fifolog_int_open(&f->ff, fn, 1); 140176998Sphk if (es != NULL) 141176998Sphk return (es); 142176998Sphk es = fifolog_int_findend(f->ff, &o); 143176998Sphk if (es != NULL) 144176998Sphk return (es); 145188007Sphk i = fifolog_int_read(f->ff, o); 146188007Sphk if (i) 147188007Sphk return ("Read error, looking for seq"); 148188007Sphk f->seq = be32dec(f->ff->recbuf); 149188007Sphk if (f->seq == 0) { 150188007Sphk /* Empty fifolog */ 151188007Sphk f->seq = random(); 152176998Sphk } else { 153176998Sphk f->recno = o + 1; 154188007Sphk f->seq++; 155176998Sphk } 156176998Sphk 157216257Sphk f->obufsize = f->ff->recsize; 158216257Sphk ALLOC(&f->obuf, f->obufsize); 159216257Sphk 160219095Sphk f->ibufsize = f->obufsize * 10; 161219095Sphk ALLOC(&f->ibuf, f->ibufsize); 162219095Sphk f->ibufptr = 0; 163219095Sphk 164176998Sphk i = deflateInit(f->ff->zs, (int)f->compression); 165176998Sphk assert(i == Z_OK); 166176998Sphk 167176998Sphk f->flag |= FIFOLOG_FLG_RESTART; 168216257Sphk f->flag |= FIFOLOG_FLG_SYNC; 169216257Sphk f->ff->zs->next_out = f->obuf + 9; 170216257Sphk f->ff->zs->avail_out = f->obufsize - 9; 171176998Sphk 172176998Sphk time(&now); 173176998Sphk f->starttime = now; 174216257Sphk f->lastsync = now; 175216257Sphk f->lastwrite = now; 176176998Sphk 177176998Sphk fifolog_write_assert(f); 178176998Sphk return (NULL); 179176998Sphk} 180176998Sphk 181219095Sphk/********************************************************************** 182219095Sphk * Write an output record 183219095Sphk * Returns -1 if there are trouble writing data 184219095Sphk */ 185219095Sphk 186216257Sphkstatic int 187216257Sphkfifolog_write_output(struct fifolog_writer *f, int fl, time_t now) 188176998Sphk{ 189216257Sphk long h, l = f->ff->zs->next_out - f->obuf; 190219095Sphk ssize_t i, w; 191219095Sphk int retval = 0; 192176998Sphk 193216257Sphk h = 4; /* seq */ 194216257Sphk be32enc(f->obuf, f->seq); 195216257Sphk f->obuf[h] = f->flag; 196216257Sphk h += 1; /* flag */ 197216257Sphk if (f->flag & FIFOLOG_FLG_SYNC) { 198216257Sphk be32enc(f->obuf + h, now); 199216257Sphk h += 4; /* timestamp */ 200216257Sphk } 201176998Sphk 202219095Sphk assert(l <= (long)f->ff->recsize); /* NB: l includes h */ 203216257Sphk assert(l >= h); 204219095Sphk 205219095Sphk /* We will never write an entirely empty buffer */ 206216257Sphk if (l == h) 207216257Sphk return (0); 208216257Sphk 209219095Sphk if (l < (long)f->ff->recsize && fl == Z_NO_FLUSH) 210216257Sphk return (0); 211216257Sphk 212216257Sphk w = f->ff->recsize - l; 213216257Sphk if (w > 255) { 214216257Sphk be32enc(f->obuf + f->ff->recsize - 4, w); 215216257Sphk f->obuf[4] |= FIFOLOG_FLG_4BYTE; 216216257Sphk } else if (w > 0) { 217219095Sphk f->obuf[f->ff->recsize - 1] = (uint8_t)w; 218216257Sphk f->obuf[4] |= FIFOLOG_FLG_1BYTE; 219176998Sphk } 220216257Sphk 221219095Sphk f->cnt[FIFOLOG_PT_BYTES_POST] += l - h; 222216257Sphk 223216257Sphk i = pwrite(f->ff->fd, f->obuf, f->ff->recsize, 224216257Sphk (f->recno + 1) * f->ff->recsize); 225219095Sphk if (i != f->ff->recsize) 226219095Sphk retval = -1; 227219095Sphk else 228219095Sphk retval = 1; 229216257Sphk 230176998Sphk f->cnt[FIFOLOG_PT_WRITES]++; 231219095Sphk f->cnt[FIFOLOG_PT_RUNTIME] = now - f->starttime; 232216257Sphk 233216257Sphk f->lastwrite = now; 234219095Sphk /* 235219095Sphk * We increment these even on error, so as to properly skip bad, 236219095Sphk * sectors or other light trouble. 237219095Sphk */ 238216257Sphk f->seq++; 239216257Sphk f->recno++; 240216257Sphk f->flag = 0; 241216257Sphk 242216257Sphk memset(f->obuf, 0, f->obufsize); 243216257Sphk f->ff->zs->next_out = f->obuf + 5; 244216257Sphk f->ff->zs->avail_out = f->obufsize - 5; 245219095Sphk return (retval); 246176998Sphk} 247176998Sphk 248219095Sphk/********************************************************************** 249219095Sphk * Run the compression engine 250219095Sphk * Returns -1 if there are trouble writing data 251219095Sphk */ 252219095Sphk 253219095Sphkstatic int 254219095Sphkfifolog_write_gzip(struct fifolog_writer *f, time_t now) 255176998Sphk{ 256219095Sphk int i, fl, retval = 0; 257176998Sphk 258219095Sphk assert(now != 0); 259219095Sphk if (f->cleanup || now >= (int)(f->lastsync + f->syncrate)) { 260176998Sphk f->cleanup = 0; 261176998Sphk fl = Z_FINISH; 262176998Sphk f->cnt[FIFOLOG_PT_SYNC]++; 263216257Sphk } else if (now >= (int)(f->lastwrite + f->writerate)) { 264176998Sphk fl = Z_SYNC_FLUSH; 265176998Sphk f->cnt[FIFOLOG_PT_FLUSH]++; 266219095Sphk } else if (f->ibufptr == 0) 267219095Sphk return (0); 268176998Sphk else 269176998Sphk fl = Z_NO_FLUSH; 270176998Sphk 271219095Sphk f->ff->zs->avail_in = f->ibufptr; 272219095Sphk f->ff->zs->next_in = f->ibuf; 273176998Sphk 274216257Sphk while (1) { 275176998Sphk i = deflate(f->ff->zs, fl); 276216257Sphk assert(i == Z_OK || i == Z_BUF_ERROR || i == Z_STREAM_END); 277216257Sphk 278219095Sphk i = fifolog_write_output(f, fl, now); 279219095Sphk if (i == 0) 280176998Sphk break; 281219095Sphk if (i < 0) 282219095Sphk retval = -1; 283176998Sphk } 284216257Sphk assert(f->ff->zs->avail_in == 0); 285219095Sphk f->ibufptr = 0; 286216257Sphk if (fl == Z_FINISH) { 287216257Sphk f->flag |= FIFOLOG_FLG_SYNC; 288216257Sphk f->ff->zs->next_out = f->obuf + 9; 289216257Sphk f->ff->zs->avail_out = f->obufsize - 9; 290216257Sphk f->lastsync = now; 291216257Sphk assert(Z_OK == deflateReset(f->ff->zs)); 292176998Sphk } 293219095Sphk return (retval); 294176998Sphk} 295176998Sphk 296219095Sphk/********************************************************************** 297219095Sphk * Poll to see if we need to flush out a record 298219095Sphk * Returns -1 if there are trouble writing data 299219095Sphk */ 300219095Sphk 301216257Sphkint 302216257Sphkfifolog_write_poll(struct fifolog_writer *f, time_t now) 303176998Sphk{ 304219095Sphk 305216257Sphk if (now == 0) 306216257Sphk time(&now); 307219095Sphk return (fifolog_write_gzip(f, now)); 308176998Sphk} 309176998Sphk 310219095Sphk/********************************************************************** 311219095Sphk * Attempt to write an entry into the ibuf. 312176998Sphk * Return zero if there is no space, one otherwise 313176998Sphk */ 314176998Sphk 315176998Sphkint 316219095Sphkfifolog_write_record(struct fifolog_writer *f, uint32_t id, time_t now, 317219095Sphk const void *ptr, ssize_t len) 318176998Sphk{ 319176998Sphk const unsigned char *p; 320219095Sphk uint8_t buf[9]; 321219123Sphk ssize_t bufl; 322176998Sphk 323176998Sphk fifolog_write_assert(f); 324176998Sphk assert(!(id & (FIFOLOG_TIMESTAMP|FIFOLOG_LENGTH))); 325176998Sphk assert(ptr != NULL); 326176998Sphk 327177381Sphk p = ptr; 328176998Sphk if (len == 0) { 329219095Sphk len = strlen(ptr); 330219095Sphk len++; 331176998Sphk } else { 332176998Sphk assert(len <= 255); 333176998Sphk id |= FIFOLOG_LENGTH; 334176998Sphk } 335219095Sphk assert (len > 0); 336176998Sphk 337219095Sphk /* Do a timestamp, if needed */ 338176998Sphk if (now == 0) 339176998Sphk time(&now); 340176998Sphk 341219095Sphk if (now != f->last) 342176998Sphk id |= FIFOLOG_TIMESTAMP; 343176998Sphk 344216257Sphk /* Emit instance+flag */ 345216257Sphk be32enc(buf, id); 346219095Sphk bufl = 4; 347176998Sphk 348176998Sphk if (id & FIFOLOG_TIMESTAMP) { 349219095Sphk be32enc(buf + bufl, (uint32_t)now); 350219095Sphk bufl += 4; 351176998Sphk } 352219095Sphk if (id & FIFOLOG_LENGTH) 353219095Sphk buf[bufl++] = (u_char)len; 354176998Sphk 355219095Sphk if (bufl + len + f->ibufptr > f->ibufsize) 356219095Sphk return (0); 357219095Sphk 358219095Sphk memcpy(f->ibuf + f->ibufptr, buf, bufl); 359219095Sphk f->ibufptr += bufl; 360219095Sphk memcpy(f->ibuf + f->ibufptr, p, len); 361219095Sphk f->ibufptr += len; 362219095Sphk f->cnt[FIFOLOG_PT_BYTES_PRE] += bufl + len; 363219095Sphk 364219095Sphk if (id & FIFOLOG_TIMESTAMP) 365219095Sphk f->last = now; 366176998Sphk return (1); 367176998Sphk} 368176998Sphk 369219095Sphk/********************************************************************** 370219095Sphk * Write an entry, polling the gzip/writer until success. 371176998Sphk * Long binary entries are broken into 255 byte chunks. 372219095Sphk * Returns -1 if there are problems writing data 373176998Sphk */ 374176998Sphk 375219095Sphkint 376219095Sphkfifolog_write_record_poll(struct fifolog_writer *f, uint32_t id, time_t now, 377219095Sphk const void *ptr, ssize_t len) 378176998Sphk{ 379176998Sphk u_int l; 380176998Sphk const unsigned char *p; 381219095Sphk int retval = 0; 382176998Sphk 383219095Sphk if (now == 0) 384219095Sphk time(&now); 385176998Sphk fifolog_write_assert(f); 386176998Sphk 387176998Sphk assert(!(id & (FIFOLOG_TIMESTAMP|FIFOLOG_LENGTH))); 388176998Sphk assert(ptr != NULL); 389176998Sphk 390176998Sphk if (len == 0) { 391219095Sphk if (!fifolog_write_record(f, id, now, ptr, len)) { 392219095Sphk if (fifolog_write_gzip(f, now) < 0) 393219095Sphk retval = -1; 394219095Sphk /* The string could be too long for the ibuf, so... */ 395219095Sphk if (!fifolog_write_record(f, id, now, ptr, len)) 396219095Sphk retval = -1; 397176998Sphk } 398176998Sphk } else { 399176998Sphk for (p = ptr; len > 0; len -= l, p += l) { 400176998Sphk l = len; 401176998Sphk if (l > 255) 402176998Sphk l = 255; 403219095Sphk while (!fifolog_write_record(f, id, now, p, l)) 404219095Sphk if (fifolog_write_gzip(f, now) < 0) 405219095Sphk retval = -1; 406176998Sphk } 407176998Sphk } 408219095Sphk if (fifolog_write_gzip(f, now) < 0) 409219095Sphk retval = -1; 410176998Sphk fifolog_write_assert(f); 411219095Sphk return (retval); 412176998Sphk} 413