From df8e5c5d09ae74781bb1c2a03a3344d4008cc651 Mon Sep 17 00:00:00 2001 From: Erki Date: Wed, 10 Mar 2021 01:12:47 +0200 Subject: [PATCH] Add unit testing for expected output --- .drone.yml | 20 +++++- op-finder-lib/include/OperationStorage.hpp | 4 +- op-finder-lib/src/OperationStorage.cpp | 10 ++- op-summarizer/op-summarizer.py | 39 ---------- op-summarizer/opsummarizer.py | 83 ++++++++++++++++++++++ op-summarizer/tests.py | 76 ++++++++++++++++++++ testcases/assignment.c | 24 ------- testcases/fir_expected.json | 1 + testcases/for_loop.c | 7 ++ testcases/for_loop_expected.json | 1 + testcases/gauss_blur.c | 5 +- testcases/gauss_blur_expected.json | 1 + testcases/matrix.c | 8 +-- testcases/matrix_expected.json | 1 + testcases/stuff.h | 17 ----- 15 files changed, 208 insertions(+), 89 deletions(-) delete mode 100644 op-summarizer/op-summarizer.py create mode 100644 op-summarizer/opsummarizer.py create mode 100644 op-summarizer/tests.py delete mode 100644 testcases/assignment.c create mode 100644 testcases/fir_expected.json create mode 100644 testcases/for_loop.c create mode 100644 testcases/for_loop_expected.json create mode 100644 testcases/gauss_blur_expected.json create mode 100644 testcases/matrix_expected.json delete mode 100644 testcases/stuff.h diff --git a/.drone.yml b/.drone.yml index 7202c63..4553806 100644 --- a/.drone.yml +++ b/.drone.yml @@ -7,12 +7,30 @@ steps: pull: never image: erki/llvm:latest commands: + - apt install --no-install-recommends python3-pip -y - pip3 install --upgrade conan - conan profile new default --detect - conan profile update settings.compiler.libcxx=libstdc++11 default - - mkdir build + - mkdir -p build - cd build - conan install .. --build=missing - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=ON - ninja - ctest . --output-on-failure + - name: opsummarizer integration tests + pull: never + image: erki/llvm:latest + commands: + - apt install --no-install-recommends gcovr python3-pip -y + - pip3 install --upgrade conan + - conan profile new default --detect + - conan profile update settings.compiler.libcxx=libstdc++11 default + - mkdir -p build + - cd build + - conan install .. --build=missing + - cmake .. -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=OFF + - ninja + - cp -r ../testcases ./testcases + - cd ./testcases + - mv ../op-finder ./ + - python3 ../../op-summarizer/tests.py diff --git a/op-finder-lib/include/OperationStorage.hpp b/op-finder-lib/include/OperationStorage.hpp index 096d1d3..770a2d0 100644 --- a/op-finder-lib/include/OperationStorage.hpp +++ b/op-finder-lib/include/OperationStorage.hpp @@ -23,12 +23,14 @@ public: void toStream(std::ostream& stream); void toFile(const std::string& output_filename); - void pushOperation(const std::string& filename, OperationLog&& op) override; + void pushOperation(const std::string& original_filename, OperationLog&& op) override; [[nodiscard]] const std::unordered_map>& getOperations() const; private: std::unordered_map> _operations; bool _pretty_print = false; + + std::string _convertFilepath(const std::string& original); }; #endif //C_ANALYZER_OPERATIONSTORAGE_HPP diff --git a/op-finder-lib/src/OperationStorage.cpp b/op-finder-lib/src/OperationStorage.cpp index a27447f..eab9a7c 100644 --- a/op-finder-lib/src/OperationStorage.cpp +++ b/op-finder-lib/src/OperationStorage.cpp @@ -35,8 +35,9 @@ void OperationStorage::toFile(const std::string& output_filename) } -void OperationStorage::pushOperation(const std::string& filename, OperationLog&& op) +void OperationStorage::pushOperation(const std::string& original_filename, OperationLog&& op) { + const std::string filename = _convertFilepath(original_filename); auto it = _operations.find(filename); if (it == _operations.end()) @@ -49,3 +50,10 @@ const std::unordered_map>& OperationStora { return _operations; } + +std::string OperationStorage::_convertFilepath(const std::string& original) +{ + const std::filesystem::path path = original; + + return path.filename().string(); +} diff --git a/op-summarizer/op-summarizer.py b/op-summarizer/op-summarizer.py deleted file mode 100644 index d0dcac7..0000000 --- a/op-summarizer/op-summarizer.py +++ /dev/null @@ -1,39 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, List - -from gcovreader import GCovFile, GCovLine -from opfinderreader import OperationLogReader, UniqueOperation - - -if __name__ == "__main__": - gcov = GCovFile("./data/gcov.json") - ops = OperationLogReader("./data/opfinder.json") - - gcov.read() - ops.read() - - for file_name in gcov.files: - op_counter: Dict[UniqueOperation, int] = {} - - if file_name not in ops.files: - print(f"Couldn't find {file_name} in op-finder output. Skipping.") - continue - - for gcov_line in gcov.files[file_name]: - op_lines = ops.get_lines(file_name, gcov_line.line_number) - for op_log in op_lines: - # TODO: revise this. Need a special case for for-loop clauses - # or branching in general. - if op_log.branch_number != gcov_line.branch_number: - continue - - unique_op = op_log.entry - - if unique_op in op_counter: - op_counter[unique_op] += gcov_line.count - else: - op_counter[unique_op] = gcov_line.count - - print(f"Unique operations for file {file_name}:") - for uop, count in op_counter.items(): - print(f"\t{count}: {uop}") diff --git a/op-summarizer/opsummarizer.py b/op-summarizer/opsummarizer.py new file mode 100644 index 0000000..a1be312 --- /dev/null +++ b/op-summarizer/opsummarizer.py @@ -0,0 +1,83 @@ +import json + +from dataclasses import asdict +from typing import Dict, List + +from gcovreader import GCovFile +from opfinderreader import OperationLogReader, UniqueOperation + + +class OpSummarizer: + def __init__(self, gcov_path: str, opfinder_path: str) -> None: + self.gcov = GCovFile(gcov_path) + self.ops = OperationLogReader(opfinder_path) + + self.gcov.read() + self.ops.read() + + def count_operations(self, file: str) -> Dict[UniqueOperation, int]: + if file not in self.gcov.files or file not in self.ops.files: + raise RuntimeError(f"File {file} not in both parsers.") + + op_counter: Dict[UniqueOperation, int] = {} + + for gcov_line in self.gcov.files[file]: + op_lines = self.ops.get_lines(file, gcov_line.line_number) + for op_log in op_lines: + # TODO: revise this. Need a special case for for-loop clauses + # or branching in general. + if op_log.branch_number != gcov_line.branch_number: + continue + + unique_op = op_log.entry + + if unique_op in op_counter: + op_counter[unique_op] += gcov_line.count + else: + op_counter[unique_op] = gcov_line.count + + return op_counter + + @staticmethod + def operation_count_to_json_dict(unique_ops: Dict[UniqueOperation, int]) -> List[Dict]: + out = [] + + for uo, uo_count in unique_ops.items(): + d = asdict(uo) + d["count"] = uo_count + out.append(d) + + return out + + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Merges gcovr and op-finder outputs.") + parser.add_argument("files", metavar="FILES", type=str, nargs="+", + help="The files to accumulate.") + parser.add_argument("--gcov", type=str, default="./data/gcov.json", + help="The gcovr json file to use.") + parser.add_argument("--finder", type=str, default="./data/opfinder.json", + help="The op-finder json file to use.") + parser.add_argument("--output", type=str, default=None, required=False, + help="The file to output the data to.") + args = parser.parse_args() + + summarizer = OpSummarizer(args.gcov, args.finder) + + total_count = {} + + for file_name in args.files: + ops = summarizer.count_operations(file_name) + total_count[file_name] = summarizer.operation_count_to_json_dict(ops) + + print(f"Unique operations for file {file_name}:") + for uop, count in ops.items(): + print(f"\t{count}: {uop}") + + print("---------") + + if args.output: + with open(args.output, "w") as outfile: + json.dump(total_count, outfile) diff --git a/op-summarizer/tests.py b/op-summarizer/tests.py new file mode 100644 index 0000000..cbf7bee --- /dev/null +++ b/op-summarizer/tests.py @@ -0,0 +1,76 @@ +import unittest +import subprocess +import json + +from dataclasses import asdict +from typing import Dict + +from opfinderreader import UniqueOperation +from opsummarizer import OpSummarizer + + +class Compiler: + def __init__(self, root_file: str) -> None: + self.root_file = root_file + self.gcov_file = f"{root_file}_gcov.json" + self.opfinder_file = f"{root_file}_opfinder.json" + + def compile_and_profile(self) -> None: + output_file = f"{self.root_file}.out" + input_file = f"{self.root_file}.c" + + subprocess.call(["gcc", + "-fprofile-arcs", + "-ftest-coverage", + "-o", output_file, + input_file]) + + subprocess.call([f"./{output_file}"]) + + subprocess.call(["gcovr", + "-r", ".", + "--json", + "--output", self.gcov_file]) + + def find_operations(self) -> None: + input_file = f"{self.root_file}.c" + + subprocess.call(["./op-finder", + "-o", self.opfinder_file, + input_file]) + + +class SummarizerCreatesExpectedOutput(unittest.TestCase): + def __init__(self, test_name, file_name) -> None: + super(SummarizerCreatesExpectedOutput, self).__init__(test_name) + self.root_file_name = file_name + self.compiler = Compiler(file_name) + + def setUp(self) -> None: + self.compiler.compile_and_profile() + self.compiler.find_operations() + + def _get_etalon(self) -> Dict: + filename = f"{self.root_file_name}_expected.json" + with open(filename, "r") as f: + return json.load(f)[f"{self.root_file_name}.c"] + + def test_summarizer_output(self) -> None: + summarizer = OpSummarizer(self.compiler.gcov_file, self.compiler.opfinder_file) + found = summarizer.count_operations(f"{self.root_file_name}.c") + found = summarizer.operation_count_to_json_dict(found) + + etalon = self._get_etalon() + + self.assertEqual(found, etalon, msg="Found operations doesn't match etalon dictionary.") + + +if __name__ == "__main__": + suite = unittest.TestSuite() + + suite.addTest(SummarizerCreatesExpectedOutput("test_summarizer_output", "matrix")) + suite.addTest(SummarizerCreatesExpectedOutput("test_summarizer_output", "gauss_blur")) + suite.addTest(SummarizerCreatesExpectedOutput("test_summarizer_output", "for_loop")) + suite.addTest(SummarizerCreatesExpectedOutput("test_summarizer_output", "fir")) + + unittest.TextTestRunner(verbosity=2).run(suite) diff --git a/testcases/assignment.c b/testcases/assignment.c deleted file mode 100644 index 52cbcd6..0000000 --- a/testcases/assignment.c +++ /dev/null @@ -1,24 +0,0 @@ -#define A (unsigned long)4 -#define STUFF int c = a + b - -#warning butts - -#include "stuff.h" - -void g() -{ - int a = 4; - int b = 6; - - int c = a + b; - (void)c; -} - -int main() -{ - short a = A; - char b = 3; - - STUFF; - int c2 = a + A; -} diff --git a/testcases/fir_expected.json b/testcases/fir_expected.json new file mode 100644 index 0000000..2132f5e --- /dev/null +++ b/testcases/fir_expected.json @@ -0,0 +1 @@ +{"fir.c": [{"operation_name": "<", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 324}, {"operation_name": "++", "type_lhs": "int", "type_rhs": "", "type_result": "int", "count": 324}, {"operation_name": "=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 37}, {"operation_name": "=", "type_lhs": "volatile float", "type_rhs": "float", "type_result": "float", "count": 360}, {"operation_name": "/", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 360}, {"operation_name": "+", "type_lhs": "float", "type_rhs": "float", "type_result": "float", "count": 324}, {"operation_name": "*", "type_lhs": "float", "type_rhs": "float", "type_result": "float", "count": 324}, {"operation_name": "subscript", "type_lhs": "const float *", "type_rhs": "int", "type_result": "const float", "count": 324}, {"operation_name": "+", "type_lhs": "unsigned int", "type_rhs": "unsigned int", "type_result": "unsigned int", "count": 288}, {"operation_name": "subscript", "type_lhs": "const unsigned int *", "type_rhs": "int", "type_result": "const unsigned int", "count": 612}, {"operation_name": "-", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 288}, {"operation_name": "+", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 612}, {"operation_name": "subscript", "type_lhs": "volatile float *", "type_rhs": "int", "type_result": "volatile float", "count": 36}]} \ No newline at end of file diff --git a/testcases/for_loop.c b/testcases/for_loop.c new file mode 100644 index 0000000..f3937ca --- /dev/null +++ b/testcases/for_loop.c @@ -0,0 +1,7 @@ +int main() +{ + int i; + for (i = 0; i < 0; i++) + { + } +} diff --git a/testcases/for_loop_expected.json b/testcases/for_loop_expected.json new file mode 100644 index 0000000..f177ac0 --- /dev/null +++ b/testcases/for_loop_expected.json @@ -0,0 +1 @@ +{"for_loop.c": [{"operation_name": "<", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 0}, {"operation_name": "++", "type_lhs": "int", "type_rhs": "", "type_result": "int", "count": 0}, {"operation_name": "=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 1}]} \ No newline at end of file diff --git a/testcases/gauss_blur.c b/testcases/gauss_blur.c index f39f707..5371042 100644 --- a/testcases/gauss_blur.c +++ b/testcases/gauss_blur.c @@ -1,7 +1,8 @@ -#include +//#include +// TODO: stddef.h not found. -int gauss_blur(){ +int main(){ diff --git a/testcases/gauss_blur_expected.json b/testcases/gauss_blur_expected.json new file mode 100644 index 0000000..82e3a2a --- /dev/null +++ b/testcases/gauss_blur_expected.json @@ -0,0 +1 @@ +{"gauss_blur.c": [{"operation_name": "=", "type_lhs": "unsigned char *", "type_rhs": "unsigned char *", "type_result": "unsigned char *", "count": 4}, {"function_name": "malloc", "call_result_type": "void *", "count": 6}, {"operation_name": "*", "type_lhs": "unsigned long", "type_rhs": "unsigned long", "type_result": "unsigned long", "count": 6}, {"operation_name": "*", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 36868}, {"operation_name": "=", "type_lhs": "unsigned char **", "type_rhs": "unsigned char **", "type_result": "unsigned char **", "count": 2}, {"operation_name": "=", "type_lhs": "unsigned char", "type_rhs": "unsigned char", "type_result": "unsigned char", "count": 5000}, {"operation_name": "subscript", "type_lhs": "unsigned char *", "type_rhs": "int", "type_result": "unsigned char", "count": 41864}, {"operation_name": "==", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 18}, {"operation_name": "<", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 18824}, {"operation_name": "++", "type_lhs": "int", "type_rhs": "", "type_result": "int", "count": 69512}, {"operation_name": "=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 18820}, {"operation_name": "-", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 50688}, {"operation_name": "<=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 50688}, {"operation_name": "-", "type_lhs": "int", "type_rhs": "", "type_result": "int", "count": 18432}, {"operation_name": "||", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 36864}, {"operation_name": "!=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 73728}, {"operation_name": "=", "type_lhs": "char", "type_rhs": "char", "type_result": "char", "count": 36864}, {"function_name": "abs", "call_result_type": "int", "count": 36864}, {"operation_name": "+", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 110592}, {"operation_name": ">", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 8680}, {"function_name": "free", "call_result_type": "void", "count": 4}]} \ No newline at end of file diff --git a/testcases/matrix.c b/testcases/matrix.c index bd9832a..d46c831 100644 --- a/testcases/matrix.c +++ b/testcases/matrix.c @@ -17,12 +17,12 @@ int main(void) { int m, n, p; volatile UInt16 m3[3][5]; -for(m = 0; m < 3; m++) +for(m = 0; m < 3; m++) // 1 { -for(p = 0; p < 5; p++) +for(p = 0; p < 5; p++) // 3 { -m3[m][p] = 0; -for(n = 0; n < 4; n++) +m3[m][p] = 0; // 15: BasicOperation(operation_name='=', type_lhs='unsigned short', type_rhs='unsigned short', type_result='unsigned short') +for(n = 0; n < 4; n++) // 15 { m3[m][p] += m1[m][n] * m2[n][p]; } diff --git a/testcases/matrix_expected.json b/testcases/matrix_expected.json new file mode 100644 index 0000000..3042fa1 --- /dev/null +++ b/testcases/matrix_expected.json @@ -0,0 +1 @@ +{"matrix.c": [{"operation_name": "<", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 78}, {"operation_name": "++", "type_lhs": "int", "type_rhs": "", "type_result": "int", "count": 78}, {"operation_name": "=", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 19}, {"operation_name": "=", "type_lhs": "unsigned short", "type_rhs": "unsigned short", "type_result": "unsigned short", "count": 15}, {"operation_name": "subscript", "type_lhs": "volatile UInt16 *", "type_rhs": "int", "type_result": "unsigned short", "count": 75}, {"operation_name": "subscript", "type_lhs": "volatile UInt16 (*)[5]", "type_rhs": "int", "type_result": "volatile UInt16 [5]", "count": 75}, {"operation_name": "*", "type_lhs": "int", "type_rhs": "int", "type_result": "int", "count": 60}, {"operation_name": "subscript", "type_lhs": "const UInt16 *", "type_rhs": "int", "type_result": "unsigned short", "count": 120}, {"operation_name": "subscript", "type_lhs": "const UInt16 (*)[4]", "type_rhs": "int", "type_result": "const UInt16 [4]", "count": 60}, {"operation_name": "subscript", "type_lhs": "const UInt16 (*)[5]", "type_rhs": "int", "type_result": "const UInt16 [5]", "count": 60}]} \ No newline at end of file diff --git a/testcases/stuff.h b/testcases/stuff.h deleted file mode 100644 index 649f645..0000000 --- a/testcases/stuff.h +++ /dev/null @@ -1,17 +0,0 @@ -// -// Created by erki on 23.02.21. -// - -#ifndef C_ANALYZER_STUFF_H -#define C_ANALYZER_STUFF_H - -void f() -{ - int a = 4; - int b = 6; - - int c = a + b; - (void)c; -} - -#endif //C_ANALYZER_STUFF_H