mirror/naiveproxy
1
0
Fork 0
mirror of https://github.com/klzgrad/naiveproxy.git synced 2024-11-28 08:16:09 +03:00
Code Issues Projects Releases Wiki Activity
7d26435067
naiveproxy/base/test/launcher/test_results_tracker.cc
klzgrad 46caed9b9c Import chromium-70.0.3538.110
2018-12-09 21:59:24 -05:00

542 lines
20 KiB
C++
Raw Blame History

// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "base/test/launcher/test_results_tracker.h"
#include <stddef.h>
#include <memory>
#include <utility>
#include "base/base64.h"
#include "base/command_line.h"
#include "base/files/file.h"
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/format_macros.h"
#include "base/json/json_writer.h"
#include "base/json/string_escape.h"
#include "base/logging.h"
#include "base/strings/string_util.h"
#include "base/strings/stringprintf.h"
#include "base/test/gtest_util.h"
#include "base/test/launcher/test_launcher.h"
#include "base/time/time.h"
#include "base/values.h"
namespace base {
namespace {
// The default output file for XML output.
const FilePath::CharType kDefaultOutputFile[] = FILE_PATH_LITERAL(
"test_detail.xml");
// Converts the given epoch time in milliseconds to a date string in the ISO
// 8601 format, without the timezone information.
// TODO(xyzzyz): Find a good place in Chromium to put it and refactor all uses
// to point to it.
std::string FormatTimeAsIso8601(Time time) {
Time::Exploded exploded;
time.UTCExplode(&exploded);
return StringPrintf("%04d-%02d-%02dT%02d:%02d:%02d",
exploded.year,
exploded.month,
exploded.day_of_month,
exploded.hour,
exploded.minute,
exploded.second);
}
struct TestSuiteResultsAggregator {
TestSuiteResultsAggregator()
: tests(0), failures(0), disabled(0), errors(0) {}
void Add(const TestResult& result) {
tests++;
elapsed_time += result.elapsed_time;
switch (result.status) {
case TestResult::TEST_SUCCESS:
break;
case TestResult::TEST_FAILURE:
failures++;
break;
case TestResult::TEST_EXCESSIVE_OUTPUT:
case TestResult::TEST_FAILURE_ON_EXIT:
case TestResult::TEST_TIMEOUT:
case TestResult::TEST_CRASH:
case TestResult::TEST_UNKNOWN:
errors++;
break;
case TestResult::TEST_SKIPPED:
disabled++;
break;
}
}
int tests;
int failures;
int disabled;
int errors;
TimeDelta elapsed_time;
};
} // namespace
TestResultsTracker::TestResultsTracker() : iteration_(-1), out_(nullptr) {}
TestResultsTracker::~TestResultsTracker() {
DCHECK(thread_checker_.CalledOnValidThread());
if (!out_)
return;
// Maps test case names to test results.
typedef std::map<std::string, std::vector<TestResult> > TestCaseMap;
TestCaseMap test_case_map;
TestSuiteResultsAggregator all_tests_aggregator;
for (const PerIterationData::ResultsMap::value_type& i
: per_iteration_data_[iteration_].results) {
// Use the last test result as the final one.
TestResult result = i.second.test_results.back();
test_case_map[result.GetTestCaseName()].push_back(result);
all_tests_aggregator.Add(result);
}
fprintf(out_, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
fprintf(out_,
"<testsuites name=\"AllTests\" tests=\"%d\" failures=\"%d\""
" disabled=\"%d\" errors=\"%d\" time=\"%.3f\" timestamp=\"%s\">\n",
all_tests_aggregator.tests, all_tests_aggregator.failures,
all_tests_aggregator.disabled, all_tests_aggregator.errors,
all_tests_aggregator.elapsed_time.InSecondsF(),
FormatTimeAsIso8601(Time::Now()).c_str());
for (const TestCaseMap::value_type& i : test_case_map) {
const std::string testsuite_name = i.first;
const std::vector<TestResult>& results = i.second;
TestSuiteResultsAggregator aggregator;
for (const TestResult& result : results) {
aggregator.Add(result);
}
fprintf(out_,
" <testsuite name=\"%s\" tests=\"%d\" "
"failures=\"%d\" disabled=\"%d\" errors=\"%d\" time=\"%.3f\" "
"timestamp=\"%s\">\n",
testsuite_name.c_str(), aggregator.tests, aggregator.failures,
aggregator.disabled, aggregator.errors,
aggregator.elapsed_time.InSecondsF(),
FormatTimeAsIso8601(Time::Now()).c_str());
for (const TestResult& result : results) {
fprintf(out_, " <testcase name=\"%s\" status=\"run\" time=\"%.3f\""
" classname=\"%s\">\n",
result.GetTestName().c_str(),
result.elapsed_time.InSecondsF(),
result.GetTestCaseName().c_str());
if (result.status != TestResult::TEST_SUCCESS) {
// The actual failure message is not propagated up to here, as it's too
// much work to escape it properly, and in case of failure, almost
// always one needs to look into full log anyway.
fprintf(out_, " <failure message=\"\" type=\"\"></failure>\n");
}
fprintf(out_, " </testcase>\n");
}
fprintf(out_, " </testsuite>\n");
}
fprintf(out_, "</testsuites>\n");
fclose(out_);
}
bool TestResultsTracker::Init(const CommandLine& command_line) {
DCHECK(thread_checker_.CalledOnValidThread());
// Prevent initializing twice.
if (out_) {
NOTREACHED();
return false;
}
if (!command_line.HasSwitch(kGTestOutputFlag))
return true;
std::string flag = command_line.GetSwitchValueASCII(kGTestOutputFlag);
size_t colon_pos = flag.find(':');
FilePath path;
if (colon_pos != std::string::npos) {
FilePath flag_path =
command_line.GetSwitchValuePath(kGTestOutputFlag);
FilePath::StringType path_string = flag_path.value();
path = FilePath(path_string.substr(colon_pos + 1));
// If the given path ends with '/', consider it is a directory.
// Note: This does NOT check that a directory (or file) actually exists
// (the behavior is same as what gtest does).
if (path.EndsWithSeparator()) {
FilePath executable = command_line.GetProgram().BaseName();
path = path.Append(executable.ReplaceExtension(
FilePath::StringType(FILE_PATH_LITERAL("xml"))));
}
}
if (path.value().empty())
path = FilePath(kDefaultOutputFile);
FilePath dir_name = path.DirName();
if (!DirectoryExists(dir_name)) {
LOG(WARNING) << "The output directory does not exist. "
<< "Creating the directory: " << dir_name.value();
// Create the directory if necessary (because the gtest does the same).
if (!CreateDirectory(dir_name)) {
LOG(ERROR) << "Failed to created directory " << dir_name.value();
return false;
}
}
out_ = OpenFile(path, "w");
if (!out_) {
LOG(ERROR) << "Cannot open output file: "
<< path.value() << ".";
return false;
}
return true;
}
void TestResultsTracker::OnTestIterationStarting() {
DCHECK(thread_checker_.CalledOnValidThread());
// Start with a fresh state for new iteration.
iteration_++;
per_iteration_data_.push_back(PerIterationData());
}
void TestResultsTracker::AddTest(const std::string& test_name) {
// Record disabled test names without DISABLED_ prefix so that they are easy
// to compare with regular test names, e.g. before or after disabling.
all_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
}
void TestResultsTracker::AddDisabledTest(const std::string& test_name) {
// Record disabled test names without DISABLED_ prefix so that they are easy
// to compare with regular test names, e.g. before or after disabling.
disabled_tests_.insert(TestNameWithoutDisabledPrefix(test_name));
}
void TestResultsTracker::AddTestLocation(const std::string& test_name,
const std::string& file,
int line) {
test_locations_.insert(std::make_pair(test_name, CodeLocation(file, line)));
}
void TestResultsTracker::AddTestResult(const TestResult& result) {
DCHECK(thread_checker_.CalledOnValidThread());
// Record disabled test names without DISABLED_ prefix so that they are easy
// to compare with regular test names, e.g. before or after disabling.
per_iteration_data_[iteration_].results[
TestNameWithoutDisabledPrefix(result.full_name)].test_results.push_back(
result);
}
void TestResultsTracker::PrintSummaryOfCurrentIteration() const {
TestStatusMap tests_by_status(GetTestStatusMapForCurrentIteration());
PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
tests_by_status[TestResult::TEST_FAILURE].end(),
"failed");
PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
"failed on exit");
PrintTests(tests_by_status[TestResult::TEST_EXCESSIVE_OUTPUT].begin(),
tests_by_status[TestResult::TEST_EXCESSIVE_OUTPUT].end(),
"produced excessive output");
PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
tests_by_status[TestResult::TEST_TIMEOUT].end(),
"timed out");
PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
tests_by_status[TestResult::TEST_CRASH].end(),
"crashed");
PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
tests_by_status[TestResult::TEST_SKIPPED].end(),
"skipped");
PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
tests_by_status[TestResult::TEST_UNKNOWN].end(),
"had unknown result");
}
void TestResultsTracker::PrintSummaryOfAllIterations() const {
DCHECK(thread_checker_.CalledOnValidThread());
TestStatusMap tests_by_status(GetTestStatusMapForAllIterations());
fprintf(stdout, "Summary of all test iterations:\n");
fflush(stdout);
PrintTests(tests_by_status[TestResult::TEST_FAILURE].begin(),
tests_by_status[TestResult::TEST_FAILURE].end(),
"failed");
PrintTests(tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].begin(),
tests_by_status[TestResult::TEST_FAILURE_ON_EXIT].end(),
"failed on exit");
PrintTests(tests_by_status[TestResult::TEST_EXCESSIVE_OUTPUT].begin(),
tests_by_status[TestResult::TEST_EXCESSIVE_OUTPUT].end(),
"produced excessive output");
PrintTests(tests_by_status[TestResult::TEST_TIMEOUT].begin(),
tests_by_status[TestResult::TEST_TIMEOUT].end(),
"timed out");
PrintTests(tests_by_status[TestResult::TEST_CRASH].begin(),
tests_by_status[TestResult::TEST_CRASH].end(),
"crashed");
PrintTests(tests_by_status[TestResult::TEST_SKIPPED].begin(),
tests_by_status[TestResult::TEST_SKIPPED].end(),
"skipped");
PrintTests(tests_by_status[TestResult::TEST_UNKNOWN].begin(),
tests_by_status[TestResult::TEST_UNKNOWN].end(),
"had unknown result");
fprintf(stdout, "End of the summary.\n");
fflush(stdout);
}
void TestResultsTracker::AddGlobalTag(const std::string& tag) {
global_tags_.insert(tag);
}
bool TestResultsTracker::SaveSummaryAsJSON(
const FilePath& path,
const std::vector<std::string>& additional_tags) const {
std::unique_ptr<DictionaryValue> summary_root(new DictionaryValue);
std::unique_ptr<ListValue> global_tags(new ListValue);
for (const auto& global_tag : global_tags_) {
global_tags->AppendString(global_tag);
}
for (const auto& tag : additional_tags) {
global_tags->AppendString(tag);
}
summary_root->Set("global_tags", std::move(global_tags));
std::unique_ptr<ListValue> all_tests(new ListValue);
for (const auto& test : all_tests_) {
all_tests->AppendString(test);
}
summary_root->Set("all_tests", std::move(all_tests));
std::unique_ptr<ListValue> disabled_tests(new ListValue);
for (const auto& disabled_test : disabled_tests_) {
disabled_tests->AppendString(disabled_test);
}
summary_root->Set("disabled_tests", std::move(disabled_tests));
std::unique_ptr<ListValue> per_iteration_data(new ListValue);
for (int i = 0; i <= iteration_; i++) {
std::unique_ptr<DictionaryValue> current_iteration_data(
new DictionaryValue);
for (PerIterationData::ResultsMap::const_iterator j =
per_iteration_data_[i].results.begin();
j != per_iteration_data_[i].results.end();
++j) {
std::unique_ptr<ListValue> test_results(new ListValue);
for (size_t k = 0; k < j->second.test_results.size(); k++) {
const TestResult& test_result = j->second.test_results[k];
std::unique_ptr<DictionaryValue> test_result_value(new DictionaryValue);
test_result_value->SetString("status", test_result.StatusAsString());
test_result_value->SetInteger(
"elapsed_time_ms",
static_cast<int>(test_result.elapsed_time.InMilliseconds()));
bool lossless_snippet = false;
if (IsStringUTF8(test_result.output_snippet)) {
test_result_value->SetString(
"output_snippet", test_result.output_snippet);
lossless_snippet = true;
} else {
test_result_value->SetString(
"output_snippet",
"<non-UTF-8 snippet, see output_snippet_base64>");
}
// TODO(phajdan.jr): Fix typo in JSON key (losless -> lossless)
// making sure not to break any consumers of this data.
test_result_value->SetBoolean("losless_snippet", lossless_snippet);
// Also include the raw version (base64-encoded so that it can be safely
// JSON-serialized - there are no guarantees about character encoding
// of the snippet). This can be very useful piece of information when
// debugging a test failure related to character encoding.
std::string base64_output_snippet;
Base64Encode(test_result.output_snippet, &base64_output_snippet);
test_result_value->SetString("output_snippet_base64",
base64_output_snippet);
std::unique_ptr<ListValue> test_result_parts(new ListValue);
for (const TestResultPart& result_part :
test_result.test_result_parts) {
std::unique_ptr<DictionaryValue> result_part_value(
new DictionaryValue);
result_part_value->SetString("type", result_part.TypeAsString());
result_part_value->SetString("file", result_part.file_name);
result_part_value->SetInteger("line", result_part.line_number);
bool lossless_summary = IsStringUTF8(result_part.summary);
if (lossless_summary) {
result_part_value->SetString("summary", result_part.summary);
} else {
result_part_value->SetString(
"summary", "<non-UTF-8 snippet, see summary_base64>");
}
result_part_value->SetBoolean("lossless_summary", lossless_summary);
std::string encoded_summary;
Base64Encode(result_part.summary, &encoded_summary);
result_part_value->SetString("summary_base64", encoded_summary);
bool lossless_message = IsStringUTF8(result_part.message);
if (lossless_message) {
result_part_value->SetString("message", result_part.message);
} else {
result_part_value->SetString(
"message", "<non-UTF-8 snippet, see message_base64>");
}
result_part_value->SetBoolean("lossless_message", lossless_message);
std::string encoded_message;
Base64Encode(result_part.message, &encoded_message);
result_part_value->SetString("message_base64", encoded_message);
test_result_parts->Append(std::move(result_part_value));
}
test_result_value->Set("result_parts", std::move(test_result_parts));
test_results->Append(std::move(test_result_value));
}
current_iteration_data->SetWithoutPathExpansion(j->first,
std::move(test_results));
}
per_iteration_data->Append(std::move(current_iteration_data));
}
summary_root->Set("per_iteration_data", std::move(per_iteration_data));
std::unique_ptr<DictionaryValue> test_locations(new DictionaryValue);
for (const auto& item : test_locations_) {
std::string test_name = item.first;
CodeLocation location = item.second;
std::unique_ptr<DictionaryValue> location_value(new DictionaryValue);
location_value->SetString("file", location.file);
location_value->SetInteger("line", location.line);
test_locations->SetWithoutPathExpansion(test_name,
std::move(location_value));
}
summary_root->Set("test_locations", std::move(test_locations));
std::string json;
if (!JSONWriter::Write(*summary_root, &json))
return false;
File output(path, File::FLAG_CREATE_ALWAYS | File::FLAG_WRITE);
if (!output.IsValid())
return false;
int json_size = static_cast<int>(json.size());
if (output.WriteAtCurrentPos(json.data(), json_size) != json_size) {
return false;
}
// File::Flush() will call fsync(). This is important on Fuchsia to ensure
// that the file is written to the disk - the system running under qemu will
// shutdown shortly after the test completes. On Fuchsia fsync() times out
// after 15 seconds. Apparently this may not be enough in some cases,
// particularly when running net_unittests on buildbots, see
// https://crbug.com/796318. Try calling fsync() more than once to workaround
// this issue.
//
// TODO(sergeyu): Figure out a better solution.
int flush_attempts_left = 4;
while (flush_attempts_left-- > 0) {
if (output.Flush())
return true;
LOG(ERROR) << "fsync() failed when saving test output summary. "
<< ((flush_attempts_left > 0) ? "Retrying." : " Giving up.");
}
return false;
}
TestResultsTracker::TestStatusMap
TestResultsTracker::GetTestStatusMapForCurrentIteration() const {
TestStatusMap tests_by_status;
GetTestStatusForIteration(iteration_, &tests_by_status);
return tests_by_status;
}
TestResultsTracker::TestStatusMap
TestResultsTracker::GetTestStatusMapForAllIterations() const {
TestStatusMap tests_by_status;
for (int i = 0; i <= iteration_; i++)
GetTestStatusForIteration(i, &tests_by_status);
return tests_by_status;
}
void TestResultsTracker::GetTestStatusForIteration(
int iteration, TestStatusMap* map) const {
for (PerIterationData::ResultsMap::const_iterator j =
per_iteration_data_[iteration].results.begin();
j != per_iteration_data_[iteration].results.end();
++j) {
// Use the last test result as the final one.
const TestResult& result = j->second.test_results.back();
(*map)[result.status].insert(result.full_name);
}
}
// Utility function to print a list of test names. Uses iterator to be
// compatible with different containers, like vector and set.
template<typename InputIterator>
void TestResultsTracker::PrintTests(InputIterator first,
InputIterator last,
const std::string& description) const {
size_t count = std::distance(first, last);
if (count == 0)
return;
fprintf(stdout,
"%" PRIuS " test%s %s:\n",
count,
count != 1 ? "s" : "",
description.c_str());
for (InputIterator it = first; it != last; ++it) {
const std::string& test_name = *it;
const auto location_it = test_locations_.find(test_name);
DCHECK(location_it != test_locations_.end()) << test_name;
const CodeLocation& location = location_it->second;
fprintf(stdout, " %s (%s:%d)\n", test_name.c_str(),
location.file.c_str(), location.line);
}
fflush(stdout);
}
TestResultsTracker::AggregateTestResult::AggregateTestResult() = default;
TestResultsTracker::AggregateTestResult::AggregateTestResult(
const AggregateTestResult& other) = default;
TestResultsTracker::AggregateTestResult::~AggregateTestResult() = default;
TestResultsTracker::PerIterationData::PerIterationData() = default;
TestResultsTracker::PerIterationData::PerIterationData(
const PerIterationData& other) = default;
TestResultsTracker::PerIterationData::~PerIterationData() = default;
} // namespace base
Reference in New Issue View Git Blame Copy Permalink