lib/compress/zstd_cwksp.h

149 MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws);
151 MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) {
152     (void)ws;
153     assert(ws->workspace <= ws->objectEnd);
154     assert(ws->objectEnd <= ws->tableEnd);
155     assert(ws->objectEnd <= ws->tableValidEnd);
156     assert(ws->tableEnd <= ws->allocStart);
157     assert(ws->tableValidEnd <= ws->allocStart);
158     assert(ws->allocStart <= ws->workspaceEnd);
189         ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase) {
190     assert(phase >= ws->phase);
191     if (phase > ws->phase) {
192         if (ws->phase < ZSTD_cwksp_alloc_buffers &&
194             ws->tableValidEnd = ws->objectEnd;
196         if (ws->phase < ZSTD_cwksp_alloc_aligned &&
205             ws->allocStart = (BYTE*)ws->allocStart - ((size_t)ws->allocStart & (sizeof(U32)-1));
206             if (ws->allocStart < ws->tableValidEnd) {
207                 ws->tableValidEnd = ws->allocStart;
210         ws->phase = phase;
217 MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) {
218     return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd);
225         ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase_e phase) {
227     void* bottom = ws->tableEnd;
228     ZSTD_cwksp_internal_advance_phase(ws, phase);
229     alloc = (BYTE *)ws->allocStart - bytes;
237         alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
238     ZSTD_cwksp_assert_internal_consistency(ws);
242         ws->allocFailed = 1;
245     if (alloc < ws->tableValidEnd) {
246         ws->tableValidEnd = alloc;
248     ws->allocStart = alloc;
263 MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) {
264     return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers);
270 MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) {
272     return ZSTD_cwksp_reserve_internal(ws, ZSTD_cwksp_align(bytes, sizeof(U32)), ZSTD_cwksp_alloc_aligned);
280 MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) {
282     void* alloc = ws->tableEnd;
284     void* top = ws->allocStart;
287         alloc, bytes, ZSTD_cwksp_available_space(ws) - bytes);
289     ZSTD_cwksp_internal_advance_phase(ws, phase);
290     ZSTD_cwksp_assert_internal_consistency(ws);
294         ws->allocFailed = 1;
297     ws->tableEnd = end;
309 MEM_STATIC void* ZSTD_cwksp_reserve_object(ZSTD_cwksp* ws, size_t bytes) {
311     void* alloc = ws->objectEnd;
321         alloc, bytes, roundedBytes, ZSTD_cwksp_available_space(ws) - roundedBytes);
324     ZSTD_cwksp_assert_internal_consistency(ws);
326     if (ws->phase != ZSTD_cwksp_alloc_objects || end > ws->workspaceEnd) {
328         ws->allocFailed = 1;
331     ws->objectEnd = end;
332     ws->tableEnd = end;
333     ws->tableValidEnd = end;
345 MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) {
353         size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
354         assert(__msan_test_shadow(ws->objectEnd, size) == -1);
355         __msan_poison(ws->objectEnd, size);
359     assert(ws->tableValidEnd >= ws->objectEnd);
360     assert(ws->tableValidEnd <= ws->allocStart);
361     ws->tableValidEnd = ws->objectEnd;
362     ZSTD_cwksp_assert_internal_consistency(ws);
365 MEM_STATIC void ZSTD_cwksp_mark_tables_clean(ZSTD_cwksp* ws) {
367     assert(ws->tableValidEnd >= ws->objectEnd);
368     assert(ws->tableValidEnd <= ws->allocStart);
369     if (ws->tableValidEnd < ws->tableEnd) {
370         ws->tableValidEnd = ws->tableEnd;
372     ZSTD_cwksp_assert_internal_consistency(ws);
378 MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) {
380     assert(ws->tableValidEnd >= ws->objectEnd);
381     assert(ws->tableValidEnd <= ws->allocStart);
382     if (ws->tableValidEnd < ws->tableEnd) {
383         memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd);
385     ZSTD_cwksp_mark_tables_clean(ws);
392 MEM_STATIC void ZSTD_cwksp_clear_tables(ZSTD_cwksp* ws) {
397         size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd;
398         __asan_poison_memory_region(ws->objectEnd, size);
402     ws->tableEnd = ws->objectEnd;
403     ZSTD_cwksp_assert_internal_consistency(ws);
410 MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) {
419         size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd;
420         __msan_poison(ws->tableValidEnd, size);
426         size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->objectEnd;
427         __asan_poison_memory_region(ws->objectEnd, size);
431     ws->tableEnd = ws->objectEnd;
432     ws->allocStart = ws->workspaceEnd;
433     ws->allocFailed = 0;
434     if (ws->phase > ZSTD_cwksp_alloc_buffers) {
435         ws->phase = ZSTD_cwksp_alloc_buffers;
437     ZSTD_cwksp_assert_internal_consistency(ws);
445 MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size) {
448     ws->workspace = start;
449     ws->workspaceEnd = (BYTE*)start + size;
450     ws->objectEnd = ws->workspace;
451     ws->tableValidEnd = ws->objectEnd;
452     ws->phase = ZSTD_cwksp_alloc_objects;
453     ZSTD_cwksp_clear(ws);
454     ws->workspaceOversizedDuration = 0;
455     ZSTD_cwksp_assert_internal_consistency(ws);
458 MEM_STATIC size_t ZSTD_cwksp_create(ZSTD_cwksp* ws, size_t size, ZSTD_customMem customMem) {
462     ZSTD_cwksp_init(ws, workspace, size);
466 MEM_STATIC void ZSTD_cwksp_free(ZSTD_cwksp* ws, ZSTD_customMem customMem) {
467     void *ptr = ws->workspace;
469     memset(ws, 0, sizeof(ZSTD_cwksp));
482 MEM_STATIC size_t ZSTD_cwksp_sizeof(const ZSTD_cwksp* ws) {
483     return (size_t)((BYTE*)ws->workspaceEnd - (BYTE*)ws->workspace);
486 MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) {
487     return ws->allocFailed;
494 MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws) {
495     return (size_t)((BYTE*)ws->allocStart - (BYTE*)ws->tableEnd);
498 MEM_STATIC int ZSTD_cwksp_check_available(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
499     return ZSTD_cwksp_available_space(ws) >= additionalNeededSpace;
502 MEM_STATIC int ZSTD_cwksp_check_too_large(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
504         ws, additionalNeededSpace * ZSTD_WORKSPACETOOLARGE_FACTOR);
507 MEM_STATIC int ZSTD_cwksp_check_wasteful(ZSTD_cwksp* ws, size_t additionalNeededSpace) {
508     return ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)
509         && ws->workspaceOversizedDuration > ZSTD_WORKSPACETOOLARGE_MAXDURATION;
513         ZSTD_cwksp* ws, size_t additionalNeededSpace) {
514     if (ZSTD_cwksp_check_too_large(ws, additionalNeededSpace)) {
515         ws->workspaceOversizedDuration++;
517         ws->workspaceOversizedDuration = 0;