diff options
Diffstat (limited to 'kernel')
| -rw-r--r-- | kernel/trace/ring_buffer.c | 107 |
1 files changed, 69 insertions, 38 deletions
diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 6afd8ea5081a..c10cf4ba91d6 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -363,6 +363,7 @@ struct buffer_page { static void rb_init_page(struct buffer_data_page *bpage) { local_set(&bpage->commit, 0); + bpage->time_stamp = 0; } static __always_inline unsigned int rb_page_commit(struct buffer_page *bpage) @@ -1878,12 +1879,14 @@ static int rb_read_data_buffer(struct buffer_data_page *dpage, int tail, int cpu return events; } -static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, - struct ring_buffer_cpu_meta *meta) +static int rb_validate_buffer(struct buffer_page *bpage, int cpu, + struct ring_buffer_cpu_meta *meta, u64 prev_ts, u64 next_ts) { + struct buffer_data_page *dpage = bpage->page; unsigned long long ts; unsigned long tail; u64 delta; + int ret; /* * When a sub-buffer is recovered from a read, the commit value may @@ -1892,9 +1895,27 @@ static int rb_validate_buffer(struct buffer_data_page *dpage, int cpu, * subbuf_size is considered invalid. */ tail = local_read(&dpage->commit) & ~RB_MISSED_MASK; - if (tail > meta->subbuf_size - BUF_PAGE_HDR_SIZE) - return -1; - return rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + if (tail <= meta->subbuf_size - BUF_PAGE_HDR_SIZE) + ret = rb_read_data_buffer(dpage, tail, cpu, &ts, &delta); + else + ret = -1; + + /* + * The timestamp must be greater than @prev_ts and smaller than @next_ts. + * Since this function works in both forward (verify) and reverse (unwind) + * loop, we don't know both @prev_ts and @next_ts at the same time. + * So use the known boundary as the boundary. + */ + if (ret < 0 || (prev_ts && prev_ts > ts) || (next_ts && ts > next_ts)) { + local_set(&bpage->entries, 0); + local_set(&dpage->commit, 0); + dpage->time_stamp = prev_ts ? prev_ts : next_ts; + ret = -1; + } else { + local_set(&bpage->entries, ret); + } + + return ret; } /* If the meta data has been validated, now validate the events */ @@ -1905,6 +1926,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) unsigned long entry_bytes = 0; unsigned long entries = 0; int discarded = 0; + bool skip = false; int ret; u64 ts; int i; @@ -1915,25 +1937,35 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) orig_head = head_page = cpu_buffer->head_page; orig_reader = cpu_buffer->reader_page; - /* Do the reader page first */ - ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu, meta); + /* Do the head page first */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, 0); + if (ret < 0) { + pr_info("Ring buffer meta [%d] invalid head page detected\n", + cpu_buffer->cpu); + /* Don't bother rewinding */ + skip = true; + ts = 0; + } else { + ts = head_page->page->time_stamp; + } + + /* Do the reader page - reader must be previous to head. */ + ret = rb_validate_buffer(orig_reader, cpu_buffer->cpu, meta, 0, ts); if (ret < 0) { pr_info("Ring buffer meta [%d] invalid reader page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&orig_reader->entries, 0); - local_set(&orig_reader->page->commit, 0); } else { entries += ret; entry_bytes += rb_page_size(orig_reader); - local_set(&orig_reader->entries, ret); + ts = orig_reader->page->time_stamp; } - ts = head_page->page->time_stamp; + if (skip) + goto skip_rewind; /* - * Try to rewind the head so that we can read the pages which already + * Try to rewind the head so that we can read the pages which are already * read in the previous boot. */ if (head_page == cpu_buffer->tail_page) @@ -1946,26 +1978,27 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == cpu_buffer->tail_page) break; - /* Ensure the page has older data than head. */ - if (ts < head_page->page->time_stamp) + /* Rewind until unused page (no timestamp, no commit). */ + if (!head_page->page->time_stamp && rb_page_commit(head_page) == 0) break; - ts = head_page->page->time_stamp; - /* Ensure the page has correct timestamp and some data. */ - if (!ts || rb_page_commit(head_page) == 0) - break; - - /* Stop rewind if the page is invalid. */ - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); - if (ret < 0) - break; - - /* Recover the number of entries and update stats. */ - local_set(&head_page->entries, ret); - if (ret) - local_inc(&cpu_buffer->pages_touched); - entries += ret; - entry_bytes += rb_page_size(head_page); + /* + * Skip if the page is invalid, or its timestamp is newer than the + * previous valid page. + */ + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, 0, ts); + if (ret < 0) { + if (!discarded) + pr_info("Ring buffer meta [%d] invalid buffer page detected\n", + cpu_buffer->cpu); + discarded++; + } else { + entries += ret; + entry_bytes += rb_page_size(head_page); + if (ret > 0) + local_inc(&cpu_buffer->pages_touched); + ts = head_page->page->time_stamp; + } } if (i) pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i); @@ -2027,6 +2060,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Nothing more to do, the only page is the reader page */ goto done; } + ts = head_page->page->time_stamp; /* Iterate until finding the commit page */ for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) { @@ -2035,15 +2069,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) if (head_page == orig_reader) continue; - ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu, meta); + ret = rb_validate_buffer(head_page, cpu_buffer->cpu, meta, ts, 0); if (ret < 0) { if (!discarded) pr_info("Ring buffer meta [%d] invalid buffer page detected\n", cpu_buffer->cpu); discarded++; - /* Instead of discard whole ring buffer, discard only this sub-buffer. */ - local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); } else { /* If the buffer has content, update pages_touched */ if (ret) @@ -2051,7 +2082,7 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) entries += ret; entry_bytes += rb_page_size(head_page); - local_set(&head_page->entries, ret); + ts = head_page->page->time_stamp; } if (head_page == cpu_buffer->commit_page) break; @@ -2079,12 +2110,12 @@ static void rb_meta_validate_events(struct ring_buffer_per_cpu *cpu_buffer) /* Reset the reader page */ local_set(&cpu_buffer->reader_page->entries, 0); - local_set(&cpu_buffer->reader_page->page->commit, 0); + rb_init_page(cpu_buffer->reader_page->page); /* Reset all the subbuffers */ for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) { local_set(&head_page->entries, 0); - local_set(&head_page->page->commit, 0); + rb_init_page(head_page->page); } } |