drivers/android/binder_alloc.c

60 static size_t binder_alloc_buffer_size(struct binder_alloc *alloc,
63 	if (list_is_last(&buffer->entry, &alloc->buffers))
64 		return alloc->buffer + alloc->buffer_size - buffer->user_data;
68 static void binder_insert_free_buffer(struct binder_alloc *alloc,
71 	struct rb_node **p = &alloc->free_buffers.rb_node;
79 	new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);
83 		      alloc->pid, new_buffer_size, new_buffer);
90 		buffer_size = binder_alloc_buffer_size(alloc, buffer);
98 	rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);
102 		struct binder_alloc *alloc, struct binder_buffer *new_buffer)
104 	struct rb_node **p = &alloc->allocated_buffers.rb_node;
123 	rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);
127 		struct binder_alloc *alloc,
130 	struct rb_node *n = alloc->allocated_buffers.rb_node;
158  * @alloc:	binder_alloc for this proc
167 struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,
172 	spin_lock(&alloc->lock);
173 	buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr);
174 	spin_unlock(&alloc->lock);
193 static void binder_lru_freelist_add(struct binder_alloc *alloc,
199 	trace_binder_update_page_range(alloc, false, start, end);
205 		index = (page_addr - alloc->buffer) / PAGE_SIZE;
206 		page = &alloc->pages[index];
211 		trace_binder_free_lru_start(alloc, index);
216 		trace_binder_free_lru_end(alloc, index);
220 static int binder_install_single_page(struct binder_alloc *alloc,
227 	if (!mmget_not_zero(alloc->mm))
234 	mmap_write_lock(alloc->mm);
238 	if (!alloc->vma) {
239 		pr_err("%d: %s failed, no vma\n", alloc->pid, __func__);
246 		pr_err("%d: failed to allocate page\n", alloc->pid);
251 	ret = vm_insert_page(alloc->vma, addr, page);
254 		       alloc->pid, __func__, addr - alloc->buffer, ret);
263 	mmap_write_unlock(alloc->mm);
264 	mmput_async(alloc->mm);
268 static int binder_install_buffer_pages(struct binder_alloc *alloc,
283 		index = (page_addr - alloc->buffer) / PAGE_SIZE;
284 		page = &alloc->pages[index];
289 		trace_binder_alloc_page_start(alloc, index);
291 		ret = binder_install_single_page(alloc, page, page_addr);
295 		trace_binder_alloc_page_end(alloc, index);
302 static void binder_lru_freelist_del(struct binder_alloc *alloc,
308 	trace_binder_update_page_range(alloc, true, start, end);
314 		index = (page_addr - alloc->buffer) / PAGE_SIZE;
315 		page = &alloc->pages[index];
318 			trace_binder_alloc_lru_start(alloc, index);
323 			trace_binder_alloc_lru_end(alloc, index);
327 		if (index + 1 > alloc->pages_high)
328 			alloc->pages_high = index + 1;
332 static inline void binder_alloc_set_vma(struct binder_alloc *alloc,
336 	smp_store_release(&alloc->vma, vma);
340 		struct binder_alloc *alloc)
343 	return smp_load_acquire(&alloc->vma);
346 static void debug_no_space_locked(struct binder_alloc *alloc)
358 	for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) {
360 		buffer_size = binder_alloc_buffer_size(alloc, buffer);
367 	for (n = rb_first(&alloc->free_buffers); n; n = rb_next(n)) {
369 		buffer_size = binder_alloc_buffer_size(alloc, buffer);
383 static bool debug_low_async_space_locked(struct binder_alloc *alloc)
402 	if (alloc->free_async_space >= alloc->buffer_size / 10) {
403 		alloc->oneway_spam_detected = false;
407 	for (n = rb_first(&alloc->allocated_buffers); n != NULL;
414 		total_alloc_size += binder_alloc_buffer_size(alloc, buffer);
423 	if (num_buffers > 50 || total_alloc_size > alloc->buffer_size / 4) {
426 			      alloc->pid, pid, num_buffers, total_alloc_size);
427 		if (!alloc->oneway_spam_detected) {
428 			alloc->oneway_spam_detected = true;
437 				struct binder_alloc *alloc,
442 	struct rb_node *n = alloc->free_buffers.rb_node;
449 	if (is_async && alloc->free_async_space < size) {
452 			      alloc->pid, size);
460 		buffer_size = binder_alloc_buffer_size(alloc, buffer);
476 				   alloc->pid, size);
477 		debug_no_space_locked(alloc);
485 		buffer_size = binder_alloc_buffer_size(alloc, buffer);
491 		binder_insert_free_buffer(alloc, new_buffer);
497 		      alloc->pid, size, buffer, buffer_size);
507 	binder_lru_freelist_del(alloc, PAGE_ALIGN(buffer->user_data),
510 	rb_erase(&buffer->rb_node, &alloc->free_buffers);
513 	binder_insert_allocated_buffer_locked(alloc, buffer);
517 		alloc->free_async_space -= size;
520 			      alloc->pid, size, alloc->free_async_space);
521 		if (debug_low_async_space_locked(alloc))
555  * @alloc:              binder_alloc for this proc
568 struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
579 	if (!binder_alloc_get_vma(alloc)) {
582 				   alloc->pid);
590 				   alloc->pid, data_size, offsets_size,
600 	spin_lock(&alloc->lock);
601 	buffer = binder_alloc_new_buf_locked(alloc, next, size, is_async);
603 		spin_unlock(&alloc->lock);
611 	spin_unlock(&alloc->lock);
613 	ret = binder_install_buffer_pages(alloc, buffer, size);
615 		binder_alloc_free_buf(alloc, buffer);
632 static void binder_delete_free_buffer(struct binder_alloc *alloc,
640 	BUG_ON(alloc->buffers.next == &buffer->entry);
646 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
652 	binder_lru_freelist_add(alloc, buffer_start_page(buffer),
659 static void binder_free_buf_locked(struct binder_alloc *alloc,
664 	buffer_size = binder_alloc_buffer_size(alloc, buffer);
672 		      alloc->pid, buffer, size, buffer_size);
677 	BUG_ON(buffer->user_data < alloc->buffer);
678 	BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size);
681 		alloc->free_async_space += buffer_size;
684 			      alloc->pid, size, alloc->free_async_space);
687 	binder_lru_freelist_add(alloc, PAGE_ALIGN(buffer->user_data),
690 	rb_erase(&buffer->rb_node, &alloc->allocated_buffers);
692 	if (!list_is_last(&buffer->entry, &alloc->buffers)) {
696 			rb_erase(&next->rb_node, &alloc->free_buffers);
697 			binder_delete_free_buffer(alloc, next);
700 	if (alloc->buffers.next != &buffer->entry) {
704 			binder_delete_free_buffer(alloc, buffer);
705 			rb_erase(&prev->rb_node, &alloc->free_buffers);
709 	binder_insert_free_buffer(alloc, buffer);
714  * @alloc: binder_alloc for this proc
726  * guaranteed that the corresponding elements of @alloc->pages[]
731 static struct page *binder_alloc_get_page(struct binder_alloc *alloc,
737 		(buffer->user_data - alloc->buffer);
742 	lru_page = &alloc->pages[index];
749  * @alloc: binder_alloc for this proc
754 static void binder_alloc_clear_buf(struct binder_alloc *alloc,
757 	size_t bytes = binder_alloc_buffer_size(alloc, buffer);
765 		page = binder_alloc_get_page(alloc, buffer,
776  * @alloc:	binder_alloc for this proc
781 void binder_alloc_free_buf(struct binder_alloc *alloc,
788 	 * increase contention for the alloc->lock if clear_on_free
793 		binder_alloc_clear_buf(alloc, buffer);
796 	spin_lock(&alloc->lock);
797 	binder_free_buf_locked(alloc, buffer);
798 	spin_unlock(&alloc->lock);
803  * @alloc:	alloc structure for this proc
814 int binder_alloc_mmap_handler(struct binder_alloc *alloc,
821 	if (unlikely(vma->vm_mm != alloc->mm)) {
828 	if (alloc->buffer_size) {
833 	alloc->buffer_size = min_t(unsigned long, vma->vm_end - vma->vm_start,
837 	alloc->buffer = vma->vm_start;
839 	alloc->pages = kcalloc(alloc->buffer_size / PAGE_SIZE,
840 			       sizeof(alloc->pages[0]),
842 	if (alloc->pages == NULL) {
844 		failure_string = "alloc page array";
848 	for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
849 		alloc->pages[i].alloc = alloc;
850 		INIT_LIST_HEAD(&alloc->pages[i].lru);
856 		failure_string = "alloc buffer struct";
860 	buffer->user_data = alloc->buffer;
861 	list_add(&buffer->entry, &alloc->buffers);
863 	binder_insert_free_buffer(alloc, buffer);
864 	alloc->free_async_space = alloc->buffer_size / 2;
867 	binder_alloc_set_vma(alloc, vma);
872 	kfree(alloc->pages);
873 	alloc->pages = NULL;
875 	alloc->buffer = 0;
877 	alloc->buffer_size = 0;
883 			   alloc->pid, vma->vm_start, vma->vm_end,
889 void binder_alloc_deferred_release(struct binder_alloc *alloc)
896 	spin_lock(&alloc->lock);
897 	BUG_ON(alloc->vma);
899 	while ((n = rb_first(&alloc->allocated_buffers))) {
906 			binder_alloc_clear_buf(alloc, buffer);
909 		binder_free_buf_locked(alloc, buffer);
913 	while (!list_empty(&alloc->buffers)) {
914 		buffer = list_first_entry(&alloc->buffers,
919 		WARN_ON_ONCE(!list_empty(&alloc->buffers));
924 	if (alloc->pages) {
927 		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
930 			if (!alloc->pages[i].page_ptr)
934 						  &alloc->pages[i].lru);
937 				     __func__, alloc->pid, i,
939 			__free_page(alloc->pages[i].page_ptr);
942 		kfree(alloc->pages);
944 	spin_unlock(&alloc->lock);
945 	if (alloc->mm)
946 		mmdrop(alloc->mm);
950 		     __func__, alloc->pid, buffers, page_count);
956  * @alloc: binder_alloc for this proc
962 				  struct binder_alloc *alloc)
967 	spin_lock(&alloc->lock);
968 	for (n = rb_first(&alloc->allocated_buffers); n; n = rb_next(n)) {
972 			   buffer->user_data - alloc->buffer,
977 	spin_unlock(&alloc->lock);
983  * @alloc: binder_alloc for this proc
986 			      struct binder_alloc *alloc)
994 	spin_lock(&alloc->lock);
999 	if (binder_alloc_get_vma(alloc) != NULL) {
1000 		for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
1001 			page = &alloc->pages[i];
1010 	spin_unlock(&alloc->lock);
1012 	seq_printf(m, "  pages high watermark: %zu\n", alloc->pages_high);
1017  * @alloc: binder_alloc for this proc
1021 int binder_alloc_get_allocated_count(struct binder_alloc *alloc)
1026 	spin_lock(&alloc->lock);
1027 	for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
1029 	spin_unlock(&alloc->lock);
1036  * @alloc: binder_alloc for this proc
1039  * Clears alloc->vma to prevent new incoming transactions from
1042 void binder_alloc_vma_close(struct binder_alloc *alloc)
1044 	binder_alloc_set_vma(alloc, NULL);
1063 	struct binder_alloc *alloc = page->alloc;
1064 	struct mm_struct *mm = alloc->mm;
1074 	if (!spin_trylock(&alloc->lock))
1079 	index = page - alloc->pages;
1080 	page_addr = alloc->buffer + index * PAGE_SIZE;
1083 	if (vma && vma != binder_alloc_get_vma(alloc))
1086 	trace_binder_unmap_kernel_start(alloc, index);
1091 	trace_binder_unmap_kernel_end(alloc, index);
1094 	spin_unlock(&alloc->lock);
1098 		trace_binder_unmap_user_start(alloc, index);
1102 		trace_binder_unmap_user_end(alloc, index);
1114 	spin_unlock(&alloc->lock);
1140  * @alloc: binder_alloc for this proc
1145 void binder_alloc_init(struct binder_alloc *alloc)
1147 	alloc->pid = current->group_leader->pid;
1148 	alloc->mm = current->mm;
1149 	mmgrab(alloc->mm);
1150 	spin_lock_init(&alloc->lock);
1151 	INIT_LIST_HEAD(&alloc->buffers);
1184  * @alloc: binder_alloc for this proc
1201 static inline bool check_buffer(struct binder_alloc *alloc,
1205 	size_t buffer_size = binder_alloc_buffer_size(alloc, buffer);
1216  * @alloc: binder_alloc for this proc
1227 binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
1233 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))
1243 		page = binder_alloc_get_page(alloc, buffer,
1258 static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
1266 	if (!check_buffer(alloc, buffer, buffer_offset, bytes))
1274 		page = binder_alloc_get_page(alloc, buffer,
1289 int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
1295 	return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,
1299 int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
1305 	return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,