1 /**
2 Copyright: Copyright (c) 2017, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 #TST-plugin_mutate_report_for_human
7 
8 TODO the full test specification is not implemented.
9 */
10 module dextool_test.test_report;
11 
12 import core.time : dur;
13 
14 import dextool.plugin.mutate.backend.database.standalone;
15 import dextool.plugin.mutate.backend.database.type;
16 import dextool.plugin.mutate.backend.type;
17 
18 import dextool_test.utility;
19 
20 // dfmt off
21 
22 @("shall report a summary of the untested mutants as human readable to stdout")
23 unittest {
24     mixin(EnvSetup(globalTestdir));
25     // Arrange
26     makeDextoolAnalyze(testEnv)
27         .addInputArg(testData ~ "report_one_ror_mutation_point.cpp")
28         .run;
29     // Act
30     auto r = makeDextoolReport(testEnv, testData.dirName)
31         .addArg(["--style", "markdown"])
32         .run;
33 
34     testConsecutiveSparseOrder!SubStr([
35         "# Mutation Type",
36         "## Summary",
37         "Untested:",
38         "Alive:",
39         "Killed:",
40         "Timeout:",
41         "Total:"
42     ]).shouldBeIn(r.stdout);
43 }
44 
45 @("shall report the alive in the database as human readable to stdout")
46 unittest {
47     mixin(EnvSetup(globalTestdir));
48     // Arrange
49     makeDextoolAnalyze(testEnv)
50         .addInputArg(testData ~ "report_one_ror_mutation_point.cpp")
51         .run;
52     auto db = Database.make((testEnv.outdir ~ defaultDb).toString);
53     db.updateMutation(MutationId(1), Mutation.Status.alive, 5.dur!"msecs", null);
54 
55     // Act
56     auto r = makeDextoolReport(testEnv, testData.dirName)
57         .addArg(["--level", "alive"])
58         .addArg(["--style", "markdown"])
59         .run;
60 
61     testConsecutiveSparseOrder!SubStr([
62         "# Mutation Type",
63         "## Mutants",
64         "| From | To   | File Line:Column                                       | ID | Status |",
65         "|------|------|--------------------------------------------------------|----|--------|",
66         "| `>`  | `>=` | plugin_testdata/report_one_ror_mutation_point.cpp 6:11 | 1  | alive  |",
67         "## Alive Mutation Statistics",
68         "| Percentage | Count | From | To   |",
69         "|------------|-------|------|------|",
70         "| 100        | 1     | `>`  | `>=` |",
71         "## Summary",
72         "Mutation execution time:         5 ms",
73         "Untested:",
74         "Alive:",
75         "Killed:",
76         "Timeout:",
77         "Total:"
78     ]).shouldBeIn(r.stdout);
79 }
80 
81 @("shall report the ROR mutations in the database as gcc compiler warnings/notes with fixits to stderr")
82 unittest {
83     auto input_src = testData ~ "report_one_ror_mutation_point.cpp";
84     mixin(EnvSetup(globalTestdir));
85     makeDextoolAnalyze(testEnv)
86         .addInputArg(input_src)
87         .run;
88     auto r = makeDextoolReport(testEnv, testData.dirName)
89         .addArg(["--mutant", "ror"])
90         .addArg(["--style", "compiler"])
91         .addArg(["--level", "all"])
92         .run;
93 
94     testConsecutiveSparseOrder!SubStr([
95                       ":6:11: warning: ror: replace '>' with '>='",
96                       ":6:11: note: status:unknown id:",
97                       `fix-it:"` ~ input_src.toString ~ `":{6:11-6:12}:">="`,
98                       ":6:11: warning: ror: replace '>' with '!='",
99                       ":6:11: note: status:unknown id:",
100                       `fix-it:"` ~ input_src.toString ~ `":{6:11-6:12}:"!="`,
101                       ":6:9: warning: rorp: replace 'x > 3' with 'false'",
102                       ":6:9: note: status:unknown id:",
103                       `fix-it:"` ~ input_src.toString ~ `":{6:9-6:14}:"false"`,
104     ]).shouldBeIn(r.stderr);
105 }
106 
107 @("shall report tool integration notes with the full text for dccTrue and dccBomb")
108 unittest {
109     auto input_src = testData ~ "report_tool_integration.cpp";
110     mixin(EnvSetup(globalTestdir));
111     makeDextoolAnalyze(testEnv)
112         .addInputArg(input_src)
113         .run;
114     auto r = makeDextoolReport(testEnv, testData.dirName)
115         .addArg(["--mutant", "dcc"])
116         .addArg(["--style", "compiler"])
117         .addArg(["--level", "all"])
118         .run;
119 
120     testConsecutiveSparseOrder!SubStr([
121                       ":7:9: warning: dcr: replace 'var1_...' with 'true'",
122                       ":7:9: note: status:unknown id:",
123                       ":7:9: note: replace 'var1_long_text > 5'",
124                       `fix-it:"` ~ input_src.toString ~ `":{7:9-7:27}:"true"`,
125                       ":11:5: warning: dcc: replace 'retur...' with '*((ch...'",
126                       ":11:5: note: status:unknown id:",
127                       ":11:5: note: replace 'return true;'",
128                       ":11:5: note: with '*((char*)0)='x';break;'",
129                       `fix-it:"` ~ input_src.toString ~ `":{11:5-11:17}:"*((char*)0)='x';break;"`,
130     ]).shouldBeIn(r.stderr);
131 }
132 
133 @("shall append a line indicating that the file is mutated")
134 unittest {
135     // TODO
136 }
137 
138 @("shall report mutants as a json")
139 unittest {
140     auto input_src = testData ~ "report_tool_integration.cpp";
141     mixin(EnvSetup(globalTestdir));
142     makeDextoolAnalyze(testEnv)
143         .addInputArg(input_src)
144         .run;
145     auto r = makeDextoolReport(testEnv, testData.dirName)
146         .addArg(["--mutant", "dcc"])
147         .addArg(["--style", "json"])
148         .addArg(["--level", "all"])
149         .run;
150 
151     writelnUt(r.stdout);
152 }
153 
154 @("shall report mutants in csv format")
155 unittest {
156     //#TST-plugin_mutate_report_as_csv
157 
158     auto input_src = testData ~ "report_as_csv.cpp";
159     mixin(EnvSetup(globalTestdir));
160     makeDextoolAnalyze(testEnv)
161         .addInputArg(input_src)
162         .run;
163     auto r = makeDextoolReport(testEnv, testData.dirName)
164         .addArg(["--mutant", "dcr"])
165         .addArg(["--style", "csv"])
166         .addArg(["--level", "all"])
167         .run;
168 
169     testConsecutiveSparseOrder!SubStr([
170         `"ID","Kind","Description","Location","Comment"`,
171         `"8","dcr","'var1_long_text >5' to 'true'","plugin_testdata/report_as_csv.cpp:7:9",""`,
172         `"9","dcr","'var1_long_text >5' to 'false'","plugin_testdata/report_as_csv.cpp:7:9",""`,
173         `"27","dcr","'case 2:`,
174         `        return true;' to '/*case 2:`,
175         `        return true;*/'","plugin_testdata/report_as_csv.cpp:11:5",""`,
176     ]).shouldBeIn(r.stdout);
177 }