From patchwork Tue Nov 23 06:38:22 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: tangmeng X-Patchwork-Id: 48021 Return-Path: X-Original-To: patchwork@sourceware.org Delivered-To: patchwork@sourceware.org Received: from server2.sourceware.org (localhost [IPv6:::1]) by sourceware.org (Postfix) with ESMTP id DF6B3385802A for ; Tue, 23 Nov 2021 06:38:44 +0000 (GMT) X-Original-To: libabigail@sourceware.org Delivered-To: libabigail@sourceware.org Received: from smtpproxy21.qq.com (smtpbg702.qq.com [203.205.195.102]) by sourceware.org (Postfix) with ESMTPS id DACB43858404 for ; Tue, 23 Nov 2021 06:38:37 +0000 (GMT) DMARC-Filter: OpenDMARC Filter v1.4.1 sourceware.org DACB43858404 Authentication-Results: sourceware.org; dmarc=none (p=none dis=none) header.from=uniontech.com Authentication-Results: sourceware.org; spf=pass smtp.mailfrom=uniontech.com X-QQ-mid: bizesmtp35t1637649512t7n8rxh0 Received: from localhost.localdomain (unknown [58.240.82.166]) by esmtp6.qq.com (ESMTP) with id ; Tue, 23 Nov 2021 14:38:26 +0800 (CST) X-QQ-SSF: 0140000000200050D000000A0000000 X-QQ-FEAT: 1UpjYZE8Zpdfpkp9iPdf7cLmUeGUOZFI3q/Sdg32wf1IsEmB/n1fC+FoJTjPV HyzMLRqB3lvpPGsHFoHyiplt49R4vVdZVwAGY6M+vZyF9AeLigkVOqn+1S0HCSVimQj3gjL fCtZuVmAAt/khe8L3zILBrWWaUKOjP1veZ4FM06XXKCQuLUJbriMDxAO8PjFPR05QzXlxw7 usyyhnqlgUM0hOW2l0FBRFEyMwOBrzP4XnyQO15AkqqjMtbjGGrnds8buRzX9zXglRnfbTi UuQTaDV7eEBLL9UD0wbwNQYfOLB3UFip5qRQ4uSI/bUw+LG8nYkKIY2s/mV4pfGjWpJ1gJY cmZzVbW4ywEhO3PtvALfakp5gw0MDsS+uQP7h8eMTxfwwro+F0= X-QQ-GoodBg: 2 From: tangmeng To: libabigail@sourceware.org Subject: [PATCH v2] Standardize and improve the output of several tests Date: Tue, 23 Nov 2021 14:38:22 +0800 Message-Id: <20211123063822.12803-1-tangmeng@uniontech.com> X-Mailer: git-send-email 2.20.1 MIME-Version: 1.0 X-QQ-SENDSIZE: 520 Feedback-ID: bizesmtp:uniontech.com:qybgforeign:qybgforeign6 X-QQ-Bgrelay: 1 X-Spam-Status: No, score=-13.3 required=5.0 tests=BAYES_00, GIT_PATCH_0, KAM_DMARC_STATUS, RCVD_IN_DNSWL_NONE, RCVD_IN_MSPIKE_H3, RCVD_IN_MSPIKE_WL, SPF_HELO_PASS, SPF_PASS, TXREP autolearn=ham autolearn_force=no version=3.4.4 X-Spam-Checker-Version: SpamAssassin 3.4.4 (2020-01-24) on server2.sourceware.org X-BeenThere: libabigail@sourceware.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: Mailing list of the Libabigail project List-Unsubscribe: , List-Archive: List-Help: List-Subscribe: , Cc: tangmeng Errors-To: libabigail-bounces+patchwork=sourceware.org@sourceware.org Sender: "Libabigail" This patch updates several test harnesses to make their output show the command line of the failing tests, a brief informative summary about the number of unit tests executed, failed and executed with success. These tests now used the abigail::tests::emit_test_{summary,status_and_update_counters} functions provided in tests/test-utils.cc. * tests/test-abicompat.cc (main): Use abigail::tests::emit_test_{summary, status_and_update_counters} functions to ameliorate and standardize test output. * tests/test-abicompat.cc (main): Likewise. * tests/test-abidiff.cc (main): Likewise. * tests/test-diff-dwarf.cc (main): Likewise. * tests/test-diff-filter.cc (main): Likewise. * tests/test-diff-pkg.cc (main): Likewise. * tests/test-diff-suppr.cc (main): Likewise. * tests/test-read-dwarf.cc (main): Likewise. * tests/test-read-write.cc (main): Likewise. * tests/test-types-stability.cc (main): Likewise. Signed-off-by: tangmeng --- tests/test-abicompat.cc | 38 +++++++++++------------------------ tests/test-abidiff.cc | 16 ++++++++++++--- tests/test-diff-dwarf.cc | 15 ++++++++++++-- tests/test-diff-filter.cc | 20 ++++++++++++++---- tests/test-diff-pkg.cc | 23 +++++++++++++++------ tests/test-diff-suppr.cc | 19 +++++++++++++++--- tests/test-read-dwarf.cc | 22 +++++++++++++++----- tests/test-read-write.cc | 27 ++++++++++++++++--------- tests/test-types-stability.cc | 19 +++++++++++++++--- 9 files changed, 138 insertions(+), 61 deletions(-) diff --git a/tests/test-abicompat.cc b/tests/test-abicompat.cc index a1339eef..eca4fc07 100644 --- a/tests/test-abicompat.cc +++ b/tests/test-abicompat.cc @@ -29,6 +29,9 @@ using std::string; using std::cerr; using std::cout; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; + struct InOutSpec { const char* in_app_path; @@ -207,7 +210,7 @@ main() using abigail::tools_utils::ensure_parent_dir_created; using abigail::tools_utils::abidiff_status; - unsigned int cnt_total = 0, cnt_passed = 0, cnt_failed = 0; + unsigned int total_count = 0, passed_count = 0, failed_count = 0; string in_app_path, in_lib1_path, in_lib2_path, suppression_path, abicompat_options, ref_report_path, out_report_path, abicompat, cmd, diffcmd; @@ -261,31 +264,14 @@ main() } else is_ok = false; - - if (is_ok) - { - cout << BRIGHT_YELLOW_COLOR - << "Test Passed:" - << DEFAULT_TERMINAL_COLOR - << cmd - << std::endl; - cnt_passed++; - } - else - { - cout << BRIGHT_RED_COLOR - << "Test Failed:" - << DEFAULT_TERMINAL_COLOR - << cmd - << std::endl; - cnt_failed++; - } - cnt_total++; + emit_test_status_and_update_counters(is_ok, + cmd, + passed_count, + failed_count, + total_count); } - cout << "Summary: " << cnt_total << " tested!" - << " Test Passed: " << cnt_passed - << ", Test Failed: " << cnt_failed - << ".\n"; - return cnt_failed; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-abidiff.cc b/tests/test-abidiff.cc index 32858ece..0276478d 100644 --- a/tests/test-abidiff.cc +++ b/tests/test-abidiff.cc @@ -173,12 +173,13 @@ using abigail::comparison::translation_unit_diff_sptr; using abigail::comparison::compute_diff; using abigail::comparison::diff_context_sptr; using abigail::comparison::diff_context; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; int main(int, char*[]) { - bool is_ok = true; - + unsigned int total_count = 0, passed_count = 0, failed_count = 0; string out_path = string(abigail::tests::get_build_dir()) + "/tests/" + specs->out_path; if (!abigail::tools_utils::ensure_parent_dir_created(out_path)) @@ -190,6 +191,7 @@ main(int, char*[]) string first_in_path, second_in_path, ref_diff_path; for (InOutSpec *s = specs; s->first_in_path; ++s) { + bool is_ok = true; first_in_path = string(abigail::tests::get_src_dir()) + "/tests/" + s->first_in_path; second_in_path = @@ -270,7 +272,15 @@ main(int, char*[]) string cmd = "diff -u " + ref_diff_path + " " + out_path; if (system(cmd.c_str())) is_ok = false; + + emit_test_status_and_update_counters(is_ok, + cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-diff-dwarf.cc b/tests/test-diff-dwarf.cc index 5eaf0e21..5ecbed69 100644 --- a/tests/test-diff-dwarf.cc +++ b/tests/test-diff-dwarf.cc @@ -360,13 +360,16 @@ main() using abigail::ir::environment_sptr; using abigail::comparison::diff_context_sptr; using abigail::comparison::diff_context; + using abigail::tests::emit_test_status_and_update_counters; + using abigail::tests::emit_test_summary; - bool is_ok = true; + unsigned int total_count = 0, passed_count = 0, failed_count = 0; string in_elfv0_path, in_elfv1_path, ref_diff_report_path, out_diff_report_path; for (InOutSpec* s = in_out_specs; s->in_elfv0_path; ++s) { + bool is_ok = true; in_elfv0_path = string(get_src_dir()) + "/tests/" + s->in_elfv0_path; in_elfv1_path = string(get_src_dir()) + "/tests/" + s->in_elfv1_path; out_diff_report_path = @@ -448,7 +451,15 @@ main() "diff -u " + ref_diff_report_path + " " + out_diff_report_path; if (system(cmd.c_str())) is_ok = false; + + emit_test_status_and_update_counters(is_ok, + cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-diff-filter.cc b/tests/test-diff-filter.cc index fd36b20a..1404d2cc 100644 --- a/tests/test-diff-filter.cc +++ b/tests/test-diff-filter.cc @@ -30,6 +30,9 @@ using std::string; using std::cerr; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; + /// This is an aggregate that specifies where a test shall get its /// input from and where it shall write its ouput to. struct InOutSpec @@ -820,6 +823,7 @@ struct test_task : public abigail::workers::task { InOutSpec spec; bool is_ok; + string cmd; string diff_cmd; string error_message; @@ -844,7 +848,7 @@ struct test_task : public abigail::workers::task using abigail::tools_utils::abidiff_status; string in_elfv0_path, in_elfv1_path, - abidiff_options, abidiff, cmd, + abidiff_options, abidiff, ref_diff_report_path, out_diff_report_path; in_elfv0_path = string(get_src_dir()) + "/tests/" + spec.in_elfv0_path; @@ -918,8 +922,6 @@ main() size_t num_workers = std::min(get_number_of_threads(), num_tests); queue task_queue(num_workers); - bool is_ok = true; - for (InOutSpec* s = in_out_specs; s->in_elfv0_path; ++s) { test_task_sptr t(new test_task(*s)); @@ -937,10 +939,13 @@ main() ABG_ASSERT(completed_tasks.size() == num_tests); + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; test_task_sptr t = dynamic_pointer_cast(*ti); if (!t->is_ok) { @@ -951,7 +956,14 @@ main() if (!t->error_message.empty()) cerr << t->error_message << '\n'; } + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-diff-pkg.cc b/tests/test-diff-pkg.cc index c9e221b7..e415196c 100644 --- a/tests/test-diff-pkg.cc +++ b/tests/test-diff-pkg.cc @@ -35,6 +35,8 @@ using std::vector; using std::cerr; using abigail::tests::get_src_dir; using abigail::tools_utils::split_string; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; struct InOutSpec { @@ -777,6 +779,7 @@ struct test_task : public abigail::workers::task { InOutSpec spec; bool is_ok; + string cmd; string diff_cmd; string error_message; @@ -800,7 +803,7 @@ struct test_task : public abigail::workers::task string first_in_package_path, second_in_package_path, prog_options, - ref_abi_diff_report_path, out_abi_diff_report_path, cmd, abipkgdiff, + ref_abi_diff_report_path, out_abi_diff_report_path, diffcmd, abipkgdiff, first_in_devel_package_path, second_in_devel_package_path, suppression_path; vector first_in_debug_package_paths, second_in_debug_package_paths; @@ -912,9 +915,9 @@ struct test_task : public abigail::workers::task if (abipkgdiff_ok) { - cmd = "diff -u " + ref_abi_diff_report_path + " " + diffcmd = "diff -u " + ref_abi_diff_report_path + " " + out_abi_diff_report_path; - if (system(cmd.c_str())) + if (system(diffcmd.c_str())) is_ok = false; } else @@ -942,8 +945,6 @@ main() size_t num_workers = std::min(get_number_of_threads(), num_tests); queue task_queue(num_workers); - bool is_ok = true; - for (InOutSpec *s = in_out_specs; s->first_in_package_path; ++s) { test_task_sptr t(new test_task(*s)); @@ -960,10 +961,13 @@ main() task_queue.get_completed_tasks(); ABG_ASSERT(completed_tasks.size() == num_tests); + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; test_task_sptr t = dynamic_pointer_cast(*ti); if (!t->is_ok) { @@ -974,7 +978,14 @@ main() if (!t->error_message.empty()) cerr << t->error_message << '\n'; } + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-diff-suppr.cc b/tests/test-diff-suppr.cc index 5b4d08d3..a92d225c 100644 --- a/tests/test-diff-suppr.cc +++ b/tests/test-diff-suppr.cc @@ -29,6 +29,8 @@ using abigail::tests::get_src_dir; using abigail::tests::get_build_dir; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; /// This is an aggregate that specifies where a test shall get its /// input from and where it shall write its ouput to. @@ -2049,6 +2051,7 @@ struct test_task : public abigail::workers::task bool is_ok; std::string error_message; + std::string cmd; test_task(const InOutSpec& s, const std::string& in_base_path, @@ -2105,7 +2108,7 @@ struct test_task : public abigail::workers::task if (!headers_dir2.empty()) abidiff += " --hd2 " + in_base_path + headers_dir2; - const std::string cmd = abidiff + " " + in_elfv0_path + " " + in_elfv1_path + cmd = abidiff + " " + in_elfv0_path + " " + in_elfv1_path + " > " + out_diff_report_path; bool bidiff_ok = true; @@ -2194,12 +2197,14 @@ main(int argc, char *argv[]) ABG_ASSERT(completed_tasks.size() == num_tests); - bool is_ok = true; + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (std::vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; std::shared_ptr t = std::dynamic_pointer_cast(*ti); if (!t->is_ok) @@ -2211,7 +2216,15 @@ main(int argc, char *argv[]) std::cerr << "FAIL: test with output '" << t->spec.out_report_path << "' failed!\n"; } + + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-read-dwarf.cc b/tests/test-read-dwarf.cc index 585aca5e..60b6c0be 100644 --- a/tests/test-read-dwarf.cc +++ b/tests/test-read-dwarf.cc @@ -39,6 +39,8 @@ using abigail::xml_writer::set_type_id_style; using abigail::xml_writer::type_id_style_kind; using abigail::xml_writer::write_context_sptr; using abigail::xml_writer::write_corpus; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; /// This is an aggregate that specifies where a test shall get its /// input from, and where it shall write its ouput to. @@ -561,6 +563,7 @@ struct test_task : public abigail::workers::task string out_abi_base; string in_elf_base; string in_abi_base; + string cmd; test_task(const InOutSpec &s, string& a_out_abi_base, @@ -658,7 +661,7 @@ struct test_task : public abigail::workers::task if (!in_public_headers_path.empty()) drop_private_types += "--headers-dir " + in_public_headers_path + " --drop-private-types"; - string cmd = abidw + " " + drop_private_types + " --abidiff " + in_elf_path; + cmd = abidw + " " + drop_private_types + " --abidiff " + in_elf_path; if (system(cmd.c_str())) { error_message = string("ABIs differ:\n") @@ -670,8 +673,8 @@ struct test_task : public abigail::workers::task } in_abi_path = in_abi_base + spec.in_abi_path; - cmd = "diff -u " + in_abi_path + " " + out_abi_path; - if (system(cmd.c_str())) + string diffcmd = "diff -u " + in_abi_path + " " + out_abi_path; + if (system(diffcmd.c_str())) is_ok = false; } }; // end struct test_task @@ -706,7 +709,6 @@ main(int argc, char *argv[]) : std::min(abigail::workers::get_number_of_threads(), num_tests)); abigail::workers::queue task_queue(num_workers); - bool is_ok = true; string out_abi_base = string(get_build_dir()) + "/tests/"; string in_elf_base = string(abigail::tests::get_src_dir()) + "/tests/"; @@ -731,11 +733,14 @@ main(int argc, char *argv[]) ABG_ASSERT(completed_tasks.size() == num_tests); + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; test_task_sptr t = dynamic_pointer_cast(*ti); if (!t->is_ok) { @@ -743,7 +748,14 @@ main(int argc, char *argv[]) if (!t->error_message.empty()) cerr << t->error_message << '\n'; } + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-read-write.cc b/tests/test-read-write.cc index 54bc2b5e..fe909e82 100644 --- a/tests/test-read-write.cc +++ b/tests/test-read-write.cc @@ -45,6 +45,9 @@ using abigail::workers::task; using abigail::workers::task_sptr; using abigail::workers::get_number_of_threads; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; + /// This is an aggregate that specifies where a test shall get its /// input from, and where it shall write its ouput to. struct InOutSpec @@ -255,7 +258,7 @@ struct test_task : public abigail::workers::task InOutSpec spec; bool is_ok; string in_path, out_path, in_suppr_spec_path, ref_out_path; - string diff_cmd, error_message; + string cmd, diff_cmd, error_message; /// Constructor of the task. /// @@ -328,7 +331,7 @@ struct test_task : public abigail::workers::task string abilint = string(get_build_dir()) + "/tools/abilint"; if (!in_suppr_spec_path.empty()) abilint +=string(" --suppr ") + in_suppr_spec_path; - string cmd = abilint + " " + in_path + " > " + out_path; + cmd = abilint + " " + in_path + " > " + out_path; if (system(cmd.c_str())) { @@ -337,9 +340,8 @@ struct test_task : public abigail::workers::task is_ok = false; } - cmd = "diff -u " + ref_out_path + " " + out_path; - diff_cmd = cmd; - if (system(cmd.c_str())) + diff_cmd = "diff -u " + ref_out_path + " " + out_path; + if (system(diff_cmd.c_str())) is_ok = false; } };// end struct test_task @@ -361,9 +363,6 @@ main() size_t num_workers = std::min(get_number_of_threads(), num_tests); queue task_queue(num_workers); - bool is_ok = true; - - string in_path, out_path, in_suppr_spec_path, ref_out_path; for (InOutSpec* s = in_out_specs; s->in_path; ++s) { @@ -382,10 +381,13 @@ main() ABG_ASSERT(completed_tasks.size() == num_tests); + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; test_task_sptr t = dynamic_pointer_cast(*ti); if (!t->is_ok) { @@ -398,7 +400,14 @@ main() if (system(t->diff_cmd.c_str()) == -1) cerr << "execution of '" << t->diff_cmd << "' failed\n"; } + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; } diff --git a/tests/test-types-stability.cc b/tests/test-types-stability.cc index 2c87b647..4f5e24ad 100644 --- a/tests/test-types-stability.cc +++ b/tests/test-types-stability.cc @@ -36,6 +36,9 @@ using std::string; using std::ofstream; using std::cerr; +using abigail::tests::emit_test_status_and_update_counters; +using abigail::tests::emit_test_summary; + // A set of elf files to test type stability for. const char* elf_paths[] = { @@ -63,6 +66,7 @@ struct test_task : public abigail::workers::task const string path; const bool no_default_sizes; string error_message; + string cmd; bool is_ok; /// The constructor of the test task. @@ -88,7 +92,7 @@ struct test_task : public abigail::workers::task string abidw = string(get_build_dir()) + "/tools/abidw"; string elf_path = string(get_src_dir()) + "/tests/" + path; - string cmd = abidw + " --abidiff " + cmd = abidw + " --abidiff " + (no_default_sizes ? "--no-write-default-sizes " : "") + elf_path; if (system(cmd.c_str())) @@ -144,18 +148,27 @@ main() ABG_ASSERT(completed_tasks.size () == num_tests); - bool is_ok = true; + unsigned int total_count = 0, passed_count = 0, failed_count = 0; + for (vector::const_iterator ti = completed_tasks.begin(); ti != completed_tasks.end(); ++ti) { + bool is_ok = true; test_task_sptr t = dynamic_pointer_cast(*ti); if (!t->is_ok) { is_ok = false; cerr << t->error_message << "\n"; } + emit_test_status_and_update_counters(is_ok, + t->cmd, + passed_count, + failed_count, + total_count); } - return !is_ok; + emit_test_summary(total_count, passed_count, failed_count); + + return failed_count; }