kernel/trace/ring_buffer.c

68  * associated with the CPU it is currently executing on.  A reader may read
71  * The reader is special. For each per cpu buffer, the reader has its own
72  * reader page. When a reader has read the entire reader page, this reader
75  * Now, as long as the writer is off the reader page, the reader can do what
82  *   |reader|          RING BUFFER
93  *   |reader|          RING BUFFER
104  *   |reader|          RING BUFFER
125  * After we make this swap, the reader can hand this page off to the splice
718 	/* The reader can read an empty page, but not more than that */
1098  * The reader page is always off the ring buffer, but when the
1099  * reader finishes with a page, it needs to swap its page with
1100  * a new one from the buffer. The reader needs to take from
1106  * The reader must be careful to replace only the head page, and
1108  * ASCII art, the reader sets its old page to point to the next
1110  * the old reader page. But if the writer moves the head page
1111  * during this operation, the reader could end up with the tail.
1149  *  What the above shows is that the reader just swapped out
1150  *  the reader page with a page in the buffer, but before it
1153  *  the new page added by the reader and is about to move forward
1184  * Because the reader may move the head_page pointer, we can
1186  * the reader page). But if the next page is a header page,
1205  * The unique thing about the reader page, is that, if the
1207  * back to the reader page.
1283 	/* check if the reader took the page */
1819 	 * We don't race with the readers since we have acquired the reader
1830 	 * tail page might be on reader page, we remove the next page
1924 	 * We are holding the reader lock, so the reader page won't be swapped
1927 	 * We are going to adapt the reader page update process where:
2056 		 * Don't succeed if resizing is disabled, as a reader might be
2142 		 * Don't succeed if resizing is disabled, as a reader might be
2345 	 * The iterator could be on the reader page (it starts there).
2346 	 * But the head could have moved, since the reader was
2393 	 *  MOVED  - a reader on another CPU moved the next
2394 	 *           pointer to its reader page. Give up
2432 		 * The reader is on another CPU and just did
2444 	 * set to UPDATE. This will keep the reader from
2445 	 * swapping the head page with the reader page.
2446 	 * The reader (on another CPU) will spin till
2539 		 * and the reader will ignore it.
2552 	 * This will be used by the reader to add lost event
2632 	 * We are fighting against races between a reader that
2633 	 * could be on another CPU trying to swap its reader
2641 	 * reader page.
2646 		 * If the commit is not on the reader page, then
2669 			 * commit page could still be on the reader
3800 	 * Because the commit page may be on the reader page we
3939 	struct buffer_page *reader = cpu_buffer->reader_page;
3947 	/* Reader should exhaust content in reader page */
3948 	if (reader->read != rb_page_commit(reader))
3952 	 * If writers are committing on the reader page, knowing all
3955 	if (commit == reader)
3959 	 * If writers are committing on a page other than reader page
3967 	 * to care about there're committed data, and the reader will
3968 	 * swap reader page with head page when it is to read data.
4147 	 * if the tail is on reader_page, oldest time stamp is on the reader
4382 	struct buffer_page *reader;
4391 	reader = cpu_buffer->reader_page;
4417 		(iter->head_page == reader && commit_page == head_page &&
4486 	struct buffer_page *reader = NULL;
4499 	 * start of the reader inserts an empty page, it causes
4504 		reader = NULL;
4508 	reader = cpu_buffer->reader_page;
4511 	if (cpu_buffer->reader_page->read < rb_page_size(reader))
4516 		       cpu_buffer->reader_page->read > rb_page_size(reader)))
4520 	reader = NULL;
4529 	 * Reset the reader page to size zero.
4538 	 * Splice the empty reader page into the list around the head.
4540 	reader = rb_set_head_page(cpu_buffer);
4541 	if (!reader)
4543 	cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
4544 	cpu_buffer->reader_page->list.prev = reader->list.prev;
4551 	cpu_buffer->pages = reader->list.prev;
4553 	/* The reader page will be pointing to the new head */
4560 	 * side. Note, the reader will constantly fail the swap
4579 	ret = rb_head_page_replace(reader, cpu_buffer->reader_page);
4590 	 * Now make the new head point back to the reader page.
4592 	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
4597 	/* Finally update the reader page to the new head */
4598 	cpu_buffer->reader_page = reader;
4610 	if (reader && reader->read == 0)
4611 		cpu_buffer->read_stamp = reader->page->time_stamp;
4623 		if (likely(!reader || rb_page_write(reader) <= bsize))
4628 		/* Get the latest version of the reader write value */
4634 		reader = NULL;
4640 	 * In addition, a writer may be writing on the reader page
4648 	return reader;
4654 	struct buffer_page *reader;
4657 	reader = rb_get_reader_page(cpu_buffer);
4660 	if (RB_WARN_ON(cpu_buffer, !reader))
4714 	struct buffer_page *reader;
4729 	reader = rb_get_reader_page(cpu_buffer);
4730 	if (!reader)
4757 			*ts = rb_fix_abs_ts(*ts, reader->page->time_stamp);
5667  * the writer is off the reader page.
5685 	struct buffer_page *reader;
5716 	reader = rb_get_reader_page(cpu_buffer);
5717 	if (!reader)
5722 	read = reader->read;
5723 	commit = rb_page_commit(reader);
5746 		 * the reader page.
5779 			rpos = reader->read;
5798 		cpu_buffer->read += rb_page_entries(reader);
5799 		cpu_buffer->read_bytes += rb_page_commit(reader);
5803 		bpage = reader->page;
5804 		reader->page = data_page->data;
5805 		local_set(&reader->write, 0);
5806 		local_set(&reader->entries, 0);
5807 		reader->read = 0;
5815 		if (reader->real_end)
5816 			local_set(&bpage->commit, reader->real_end);
5969 		/* Include the reader page */
6001 		/* Free the current reader page */
6004 		/* One page was allocated for the reader page */