outqueue.c revision 292588 
1///////////////////////////////////////////////////////////////////////////////
2//
3/// \file       outqueue.c
4/// \brief      Output queue handling in multithreaded coding
5//
6//  Author:     Lasse Collin
7//
8//  This file has been put into the public domain.
9//  You can do whatever you want with this file.
10//
11///////////////////////////////////////////////////////////////////////////////
12
13#include "outqueue.h"
14
15
16/// This is to ease integer overflow checking: We may allocate up to
17/// 2 * LZMA_THREADS_MAX buffers and we need some extra memory for other
18/// data structures (that's the second /2).
19#define BUF_SIZE_MAX (UINT64_MAX / LZMA_THREADS_MAX / 2 / 2)
20
21
22static lzma_ret
23get_options(uint64_t *bufs_alloc_size, uint32_t *bufs_count,
24		uint64_t buf_size_max, uint32_t threads)
25{
26	if (threads > LZMA_THREADS_MAX || buf_size_max > BUF_SIZE_MAX)
27		return LZMA_OPTIONS_ERROR;
28
29	// The number of buffers is twice the number of threads.
30	// This wastes RAM but keeps the threads busy when buffers
31	// finish out of order.
32	//
33	// NOTE: If this is changed, update BUF_SIZE_MAX too.
34	*bufs_count = threads * 2;
35	*bufs_alloc_size = *bufs_count * buf_size_max;
36
37	return LZMA_OK;
38}
39
40
41extern uint64_t
42lzma_outq_memusage(uint64_t buf_size_max, uint32_t threads)
43{
44	uint64_t bufs_alloc_size;
45	uint32_t bufs_count;
46
47	if (get_options(&bufs_alloc_size, &bufs_count, buf_size_max, threads)
48			!= LZMA_OK)
49		return UINT64_MAX;
50
51	return sizeof(lzma_outq) + bufs_count * sizeof(lzma_outbuf)
52			+ bufs_alloc_size;
53}
54
55
56extern lzma_ret
57lzma_outq_init(lzma_outq *outq, const lzma_allocator *allocator,
58		uint64_t buf_size_max, uint32_t threads)
59{
60	uint64_t bufs_alloc_size;
61	uint32_t bufs_count;
62
63	// Set bufs_count and bufs_alloc_size.
64	return_if_error(get_options(&bufs_alloc_size, &bufs_count,
65			buf_size_max, threads));
66
67	// Allocate memory if needed.
68	if (outq->buf_size_max != buf_size_max
69			|| outq->bufs_allocated != bufs_count) {
70		lzma_outq_end(outq, allocator);
71
72#if SIZE_MAX < UINT64_MAX
73		if (bufs_alloc_size > SIZE_MAX)
74			return LZMA_MEM_ERROR;
75#endif
76
77		outq->bufs = lzma_alloc(bufs_count * sizeof(lzma_outbuf),
78				allocator);
79		outq->bufs_mem = lzma_alloc((size_t)(bufs_alloc_size),
80				allocator);
81
82		if (outq->bufs == NULL || outq->bufs_mem == NULL) {
83			lzma_outq_end(outq, allocator);
84			return LZMA_MEM_ERROR;
85		}
86	}
87
88	// Initialize the rest of the main structure. Initialization of
89	// outq->bufs[] is done when they are actually needed.
90	outq->buf_size_max = (size_t)(buf_size_max);
91	outq->bufs_allocated = bufs_count;
92	outq->bufs_pos = 0;
93	outq->bufs_used = 0;
94	outq->read_pos = 0;
95
96	return LZMA_OK;
97}
98
99
100extern void
101lzma_outq_end(lzma_outq *outq, const lzma_allocator *allocator)
102{
103	lzma_free(outq->bufs, allocator);
104	outq->bufs = NULL;
105
106	lzma_free(outq->bufs_mem, allocator);
107	outq->bufs_mem = NULL;
108
109	return;
110}
111
112
113extern lzma_outbuf *
114lzma_outq_get_buf(lzma_outq *outq)
115{
116	// Caller must have checked it with lzma_outq_has_buf().
117	assert(outq->bufs_used < outq->bufs_allocated);
118
119	// Initialize the new buffer.
120	lzma_outbuf *buf = &outq->bufs[outq->bufs_pos];
121	buf->buf = outq->bufs_mem + outq->bufs_pos * outq->buf_size_max;
122	buf->size = 0;
123	buf->finished = false;
124
125	// Update the queue state.
126	if (++outq->bufs_pos == outq->bufs_allocated)
127		outq->bufs_pos = 0;
128
129	++outq->bufs_used;
130
131	return buf;
132}
133
134
135extern bool
136lzma_outq_is_readable(const lzma_outq *outq)
137{
138	uint32_t i = outq->bufs_pos - outq->bufs_used;
139	if (outq->bufs_pos < outq->bufs_used)
140		i += outq->bufs_allocated;
141
142	return outq->bufs[i].finished;
143}
144
145
146extern lzma_ret
147lzma_outq_read(lzma_outq *restrict outq, uint8_t *restrict out,
148		size_t *restrict out_pos, size_t out_size,
149		lzma_vli *restrict unpadded_size,
150		lzma_vli *restrict uncompressed_size)
151{
152	// There must be at least one buffer from which to read.
153	if (outq->bufs_used == 0)
154		return LZMA_OK;
155
156	// Get the buffer.
157	uint32_t i = outq->bufs_pos - outq->bufs_used;
158	if (outq->bufs_pos < outq->bufs_used)
159		i += outq->bufs_allocated;
160
161	lzma_outbuf *buf = &outq->bufs[i];
162
163	// If it isn't finished yet, we cannot read from it.
164	if (!buf->finished)
165		return LZMA_OK;
166
167	// Copy from the buffer to output.
168	lzma_bufcpy(buf->buf, &outq->read_pos, buf->size,
169			out, out_pos, out_size);
170
171	// Return if we didn't get all the data from the buffer.
172	if (outq->read_pos < buf->size)
173		return LZMA_OK;
174
175	// The buffer was finished. Tell the caller its size information.
176	*unpadded_size = buf->unpadded_size;
177	*uncompressed_size = buf->uncompressed_size;
178
179	// Free this buffer for further use.
180	--outq->bufs_used;
181	outq->read_pos = 0;
182
183	return LZMA_STREAM_END;
184}
185