kernel/trace/ring_buffer.c

26 #include <linux/list.h>
66  * The ring buffer is made up of a list of pages. A separate list of pages is
327  * Note, the buffer_page list must be first. The buffer pages
331  * add flags in the list struct pointers, to make the ring buffer
335 	struct list_head list;		/* list of buffer pages */
581 static struct list_head *rb_list_head(struct list_head *list);
603 		next = rb_list_head(page->list.next);
604 		page = list_entry(next, struct buffer_page, list);
1005 		 * to wake up this list. The irq_handler will reset shortest_full
1124  * head->list->prev->next        bit 1          bit 0
1174 static struct list_head *rb_list_head(struct list_head *list)
1176 	unsigned long val = (unsigned long)list;
1190 rb_is_head_page(struct buffer_page *page, struct list_head *list)
1194 	val = (unsigned long)list->next;
1196 	if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)
1211 	struct list_head *list = page->list.prev;
1213 	return rb_list_head(list->next) != &page->list;
1219 static void rb_set_list_to_head(struct list_head *list)
1223 	ptr = (unsigned long *)&list->next;
1240 	 * Set the previous list pointer to have the HEAD flag.
1242 	rb_set_list_to_head(head->list.prev);
1245 static void rb_list_head_clear(struct list_head *list)
1247 	unsigned long *ptr = (unsigned long *)&list->next;
1253  * rb_head_page_deactivate - clears head page ptr (for free list)
1260 	/* Go through the whole list and clear any pointers found. */
1272 	struct list_head *list;
1273 	unsigned long val = (unsigned long)&head->list;
1276 	list = &prev->list;
1280 	ret = cmpxchg((unsigned long *)&list->next,
1319 	struct list_head *p = rb_list_head((*bpage)->list.next);
1321 	*bpage = list_entry(p, struct buffer_page, list);
1329 	struct list_head *list;
1336 	list = cpu_buffer->pages;
1337 	if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))
1349 			if (rb_is_head_page(page, page->list.prev)) {
1365 	unsigned long *ptr = (unsigned long *)&old->list.prev->next;
1371 	return try_cmpxchg(ptr, &val, (unsigned long)&new->list);
1524 		list_add(&bpage->list, pages);
1544 	list_for_each_entry_safe(bpage, tmp, pages, list) {
1545 		list_del_init(&bpage->list);
1565 	 * The ring buffer page list is a circular list that does not
1566 	 * start and end with a list head. All page list items point to
1619 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1627 		= list_entry(cpu_buffer->pages, struct buffer_page, list);
1654 		list_for_each_entry_safe(bpage, tmp, head, list) {
1655 			list_del_init(&bpage->list);
1658 		bpage = list_entry(head, struct buffer_page, list);
1822 	 * removed from the list. We unlink all the pages in between including
1827 	tail_page = &cpu_buffer->tail_page->list;
1839 				struct buffer_page, list);
1866 						struct buffer_page, list);
1876 				list);
1900 		 * We have already removed references to this list item, just
1928 	 * 1. We first splice the start and end of list of new pages between
1931 	 *    start of new pages list.
1932 	 * 3. Finally, we update the head->prev to the end of new list.
1947 		head_page = &hpage->list;
1963 			 * yay, we replaced the page pointer to our new list,
1965 			 * pointer to point to end of list
1986 					 list) {
1987 			list_del_init(&bpage->list);
2224 					list) {
2225 			list_del_init(&bpage->list);
2643 	if (rb_is_head_page(next_page, &tail_page->list)) {
4538 	 * Splice the empty reader page into the list around the head.
4543 	cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
4544 	cpu_buffer->reader_page->list.prev = reader->list.prev;
4551 	cpu_buffer->pages = reader->list.prev;
4554 	rb_set_list_to_head(&cpu_buffer->reader_page->list);
4592 	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
5222 		= list_entry(cpu_buffer->pages, struct buffer_page, list);
5224 	list_for_each_entry(page, cpu_buffer->pages, list) {
5231 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
5989 		/* Clear the head bit to make the link list normal to read */
5992 		/* Now walk the list and free all the old sub buffers */
5993 		list_for_each_entry_safe(bpage, tmp, cpu_buffer->pages, list) {
5994 			list_del_init(&bpage->list);
5998 		bpage = list_entry(cpu_buffer->pages, struct buffer_page, list);
6006 						     struct buffer_page, list);
6007 		list_del_init(&cpu_buffer->reader_page->list);
6009 		/* The cpu_buffer pages are a link list with no head */
6014 		/* Clear the new_pages list */
6018 			= list_entry(cpu_buffer->pages, struct buffer_page, list);
6050 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {
6051 			list_del_init(&bpage->list);