dextool.plugin.mutate.backend.report.html.page_test

1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.report.html.page_test_case;
11 
12 import logger = std.experimental.logger;
13 import std.algorithm : sort;
14 import std.array : empty;
15 import std.conv : to;
16 import std.datetime : Clock, dur, SysTime;
17 import std.format : format;
18 import std.path : buildPath;
19 import std.range : enumerate;
20 import std.stdio : File;
21 import std.traits : EnumMembers;
22 
23 import arsd.dom : Element, RawSource, Link, Document, Table;
24 import my.optional;
25 import my.path : AbsolutePath;
26 import my.set;
27 
28 import dextool.plugin.mutate.backend.database : Database, spinSql, MutationId,
29     TestCaseId, MutationStatusId, MutantInfo2;
30 import dextool.plugin.mutate.backend.report.analyzers : reportTestCaseUniqueness,
31     TestCaseUniqueness, reportTestCaseSimilarityAnalyse,
32     TestCaseSimilarityAnalyse, TestCaseClassifier, makeTestCaseClassifier, TestCaseMetadata;
33 import dextool.plugin.mutate.backend.report.html.constants : HtmlStyle = Html, DashboardCss;
34 import dextool.plugin.mutate.backend.report.html.tmpl : tmplBasicPage,
35     dashboardCss, tmplSortableTable, tmplDefaultTable;
36 import dextool.plugin.mutate.backend.report.html.utility : pathToHtmlLink, toShortDate;
37 import dextool.plugin.mutate.backend.resource;
38 import dextool.plugin.mutate.backend.type : Mutation, toString, TestCase;
39 import dextool.plugin.mutate.config : ConfigReport;
40 import dextool.plugin.mutate.type : MutationKind, ReportSection;
41 
42 @safe:
43 
44 void makeTestCases(ref Database db, string tag, Element root, ref const ConfigReport conf,
45         const(Mutation.Kind)[] kinds, TestCaseMetadata metaData, AbsolutePath testCasesDir) @trusted {
46     DashboardCss.h2(root.addChild(new Link(tag, null)).setAttribute("id", tag[1 .. $]),
47             "Test Cases");
48     auto sections = conf.reportSection.toSet;
49 
50     ReportData data;
51 
52     if (ReportSection.tc_similarity in sections)
53         data.similaritiesData = reportTestCaseSimilarityAnalyse(db, kinds, 5);
54 
55     data.addSuggestion = ReportSection.tc_suggestion in sections;
56     // 10 is magic number. feels good.
57     data.classifier = makeTestCaseClassifier(db, 10);
58 
59     const tabGroupName = "testcase_class";
60     Element[Classification] tabLink;
61 
62     { // tab links
63         auto tab = root.addChild("div").addClass("tab");
64         foreach (class_; [EnumMembers!Classification]) {
65             auto b = tab.addChild("button").addClass("tablinks")
66                 .addClass("tablinks_" ~ tabGroupName);
67             b.setAttribute("onclick", format!`openTab(event, '%s', '%s')`(class_, tabGroupName));
68             b.appendText(class_.to!string);
69             tabLink[class_] = b;
70         }
71     }
72 
73     Table[Classification] tabContent;
74     foreach (a; [EnumMembers!Classification]) {
75         auto div = root.addChild("div").addClass("tabcontent")
76             .addClass("tabcontent_" ~ tabGroupName).setAttribute("id", a.to!string);
77         if (a == Classification.Redundant)
78             div.addChild("p", format(classDescription[a], data.classifier.threshold));
79         else
80             div.addChild("p", classDescription[a]);
81         tabContent[a] = tmplSortableTable(div, ["Name", "Tests", "Killed"]);
82     }
83 
84     long[Classification] classCnt;
85     foreach (tcId; spinSql!(() => db.testCaseApi.getDetectedTestCaseIds)) {
86         const name = spinSql!(() => db.testCaseApi.getTestCaseName(tcId));
87 
88         auto reportFname = name.pathToHtmlLink;
89         auto fout = File(testCasesDir ~ reportFname, "w");
90         TestCaseSummary summary;
91         spinSql!(() {
92             // do all the heavy database interaction in a transaction to
93             // speedup by reduce locking.
94             auto t = db.transaction;
95             makeTestCasePage(db, kinds, name, tcId, (data.similaritiesData is null)
96                 ? null : data.similaritiesData.similarities.get(tcId, null),
97                 data, metaData, summary, fout);
98         });
99 
100         auto classification = classify(TestCase(name), summary, data.classifier, metaData);
101         classCnt[classification] += 1;
102 
103         auto r = tabContent[classification].appendRow;
104         {
105             auto td = r.addChild("td");
106             td.addChild("a", name).href = buildPath(HtmlStyle.testCaseDir, reportFname);
107         }
108         r.addChild("td", summary.score.to!string);
109         r.addChild("td", summary.kills.to!string);
110     }
111 
112     foreach (a; classCnt.byKeyValue) {
113         tabLink[a.key].appendText(format!" %s"(a.value));
114         if (auto c = a.key in classColor)
115             tabLink[a.key].style = *c;
116     }
117 }
118 
119 private:
120 
121 enum Classification {
122     Unique,
123     Redundant,
124     Buggy,
125     Normal
126 }
127 
128 immutable string[Classification] classDescription;
129 immutable string[Classification] classColor;
130 
131 shared static this() @trusted {
132     classDescription = cast(immutable)[
133         Classification.Unique: "kills mutants that no other test case do.",
134         Classification.Redundant: "all mutants the test case kill are also killed by %s other test cases. The test case is probably redudant and thus can be removed.",
135         Classification.Buggy
136         : "zero killed mutants. The test case is most probably incorrect. Immediatly inspect the test case.",
137         Classification.Normal: ""
138     ];
139 
140     classColor = cast(immutable)[
141         // light green
142         Classification.Unique: "background-color: #b3ff99",
143         // light orange
144         Classification.Redundant: "background-color: #ffc266",
145         // light red
146         Classification.Buggy: "background-color: #ff9980"
147     ];
148 }
149 
150 Classification classify(TestCase tc, const TestCaseSummary summary,
151         const ref TestCaseClassifier tclass, const ref TestCaseMetadata metaData) {
152     if (summary.kills == 0)
153         return Classification.Buggy;
154     if (summary.score == 1)
155         return Classification.Unique;
156 
157     if (auto v = tc in metaData.redundant) {
158         if (*v)
159             return Classification.Redundant;
160     } else if (summary.score >= tclass.threshold)
161         return Classification.Redundant;
162 
163     return Classification.Normal;
164 }
165 
166 struct ReportData {
167     TestCaseSimilarityAnalyse similaritiesData;
168     TestCaseClassifier classifier;
169 
170     bool addSuggestion;
171 }
172 
173 struct TestCaseSummary {
174     long kills;
175 
176     // min(f) where f is the number of test cases that killed a mutant.
177     // thus if a test case have one unique mutant the score is 1, none then it
178     // is the lowest of all mutant test case kills.
179     long score;
180 }
181 
182 void makeTestCasePage(ref Database db, const(Mutation.Kind)[] kinds, const string name,
183         const TestCaseId tcId, TestCaseSimilarityAnalyse.Similarity[] similarities,
184         const ReportData rdata, TestCaseMetadata metaData, ref TestCaseSummary summary, ref File out_) @system {
185     import std.path : baseName;
186     import dextool.plugin.mutate.backend.type : TestCase;
187 
188     auto doc = tmplBasicPage.dashboardCss;
189     scope (success)
190         out_.write(doc.toPrettyString);
191 
192     doc.title(format("%s %s", name, Clock.currTime));
193     doc.mainBody.setAttribute("onload", "init()");
194     doc.root.childElements("head")[0].addChild("script").addChild(new RawSource(doc, jsIndex));
195 
196     doc.mainBody.addChild("h1").appendText(name);
197     if (auto v = TestCase(name) in metaData.loc) {
198         auto p = doc.mainBody.addChild("p");
199         p.addChild("a", v.file.baseName).href = v.file;
200         if (v.line.hasValue)
201             p.appendText(" at line " ~ v.line.orElse(0u).to!string);
202     }
203     if (auto v = TestCase(name) in metaData.text)
204         doc.mainBody.addChild(new RawSource(doc, *v));
205 
206     doc.mainBody.addChild("h2").appendText("Killed");
207     addKilledMutants(db, kinds, tcId, rdata, summary, doc.mainBody);
208 
209     if (!similarities.empty) {
210         doc.mainBody.addChild("h2").appendText("Similarity");
211         addSimilarity(db, similarities, doc.mainBody);
212     }
213 }
214 
215 void addKilledMutants(ref Database db, const(Mutation.Kind)[] kinds,
216         const TestCaseId tcId, const ReportData rdata, ref TestCaseSummary summary, Element root) @system {
217     import std.algorithm : min;
218 
219     auto kills = db.testCaseApi.testCaseKilledSrcMutants(kinds, tcId);
220     summary.kills = kills.length;
221     summary.score = kills.length == 0 ? 0 : long.max;
222 
223     auto uniqueElem = root.addChild("div");
224 
225     {
226         auto p = root.addChild("p");
227         p.addChild("b", "TestCases");
228         p.appendText(": number of test cases that kill the mutant.");
229     }
230     {
231         auto p = root.addChild("p");
232         p.addChild("b", "Suggestion");
233         p.appendText(": alive mutants on the same source code location. Because they are close to a mutant that this test case killed it may be suitable to extend this test case to also kill the suggested mutant.");
234     }
235 
236     auto tbl = tmplSortableTable(root, ["Link", "TestCases"] ~ (rdata.addSuggestion
237             ? ["Suggestion"] : null) ~ ["Priority", "ExitCode", "Tested"]);
238 
239     foreach (const id; kills.sort) {
240         auto r = tbl.appendRow();
241 
242         const info = db.mutantApi.getMutantInfo(id).orElse(MutantInfo2.init);
243 
244         r.addChild("td").addChild("a", format("%s:%s", info.file,
245                 info.sloc.line)).href = format("%s#%s", buildPath("..",
246                 HtmlStyle.fileDir, pathToHtmlLink(info.file)), info.id.get);
247 
248         summary.score = min(info.tcKilled, summary.score);
249         {
250             auto td = r.addChild("td", info.tcKilled.to!string);
251             if (info.tcKilled == 1) {
252                 td.style = "background-color: lightgreen";
253             }
254         }
255 
256         if (rdata.addSuggestion) {
257             auto tds = r.addChild("td");
258             foreach (s; db.mutantApi.getSurroundingAliveMutants(id).enumerate) {
259                 // column sort in the html report do not work correctly if starting from 0.
260                 auto td = tds.addChild("a", format("%s", s.index + 1));
261                 td.href = format("%s#%s", buildPath("..", HtmlStyle.fileDir,
262                         pathToHtmlLink(info.file)), db.mutantApi.getMutationId(s.value).get);
263                 td.appendText(" ");
264             }
265         }
266 
267         r.addChild("td", info.prio.get.to!string);
268         r.addChild("td", toString(info.exitStatus));
269         r.addChild("td", info.updated.toShortDate);
270     }
271 }
272 
273 void addSimilarity(ref Database db, TestCaseSimilarityAnalyse.Similarity[] similarities,
274         Element root) @system {
275     import dextool.cachetools;
276 
277     auto getPath = nullableCache!(MutationStatusId, string, (MutationStatusId id) {
278         auto path = spinSql!(() => db.mutantApi.getPath(id)).get;
279         auto mutId = spinSql!(() => db.mutantApi.getMutationId(id)).get;
280         return format!"%s#%s"(buildPath("..", HtmlStyle.fileDir, pathToHtmlLink(path)), mutId.get);
281     })(0, 30.dur!"seconds");
282 
283     root.addChild("p", "How similary this test case is to others.");
284     {
285         auto p = root.addChild("p");
286         p.addChild("b", "Note");
287         p.appendText(": The analysis is based on the mutants that the test cases kill; thus, it is dependent on the mutation operators that are used when generating the report.");
288 
289         root.addChild("p", "The intersection column is the mutants that are killed by both the current test case and in the column Test Case.")
290             .appendText(
291                     " The difference column are the mutants that are only killed by the current test case.");
292     }
293 
294     auto tbl = tmplDefaultTable(root, [
295             "Test Case", "Similarity", "Difference", "Intersection"
296             ]);
297     foreach (const sim; similarities) {
298         auto r = tbl.appendRow();
299 
300         const name = db.testCaseApi.getTestCaseName(sim.testCase);
301         r.addChild("td").addChild("a", name).href = buildPath(name.pathToHtmlLink);
302 
303         r.addChild("td", format("%#.3s", sim.similarity));
304 
305         auto difference = r.addChild("td");
306         foreach (const mut; sim.difference) {
307             auto link = difference.addChild("a", mut.to!string);
308             link.href = getPath(mut).get;
309             difference.appendText(" ");
310         }
311 
312         auto s = r.addChild("td");
313         foreach (const mut; sim.intersection) {
314             auto link = s.addChild("a", mut.to!string);
315             link.href = getPath(mut).get;
316             s.appendText(" ");
317         }
318     }
319 }