dextool.plugin.mutate.backend.report.html.page_test

1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.report.html.page_test_case;
11 
12 import logger = std.experimental.logger;
13 import std.algorithm : sort;
14 import std.array : empty;
15 import std.conv : to;
16 import std.datetime : Clock, dur, SysTime;
17 import std.format : format;
18 import std.path : buildPath;
19 import std.range : enumerate;
20 import std.stdio : File;
21 import std.traits : EnumMembers;
22 
23 import arsd.dom : Element, RawSource, Link, Document, Table;
24 import my.optional;
25 import my.path : AbsolutePath;
26 import my.set;
27 
28 import dextool.plugin.mutate.backend.database : Database, spinSql, MutationId,
29     TestCaseId, MutationStatusId, MutantInfo2;
30 import dextool.plugin.mutate.backend.report.analyzers : reportTestCaseUniqueness,
31     TestCaseUniqueness, reportTestCaseSimilarityAnalyse,
32     TestCaseSimilarityAnalyse, TestCaseClassifier, makeTestCaseClassifier, TestCaseMetadata;
33 import dextool.plugin.mutate.backend.report.html.constants : HtmlStyle = Html, DashboardCss;
34 import dextool.plugin.mutate.backend.report.html.tmpl : tmplBasicPage,
35     dashboardCss, tmplSortableTable, tmplDefaultTable;
36 import dextool.plugin.mutate.backend.report.html.utility : pathToHtmlLink, toShortDate;
37 import dextool.plugin.mutate.backend.resource;
38 import dextool.plugin.mutate.backend.type : Mutation, toString, TestCase;
39 import dextool.plugin.mutate.config : ConfigReport;
40 import dextool.plugin.mutate.type : ReportSection;
41 import dextool.plugin.mutate.backend.report.html.utility;
42 
43 @safe:
44 
45 void makeTestCases(ref Database db, string tag, Document doc, Element root,
46         ref const ConfigReport conf, TestCaseMetadata metaData, AbsolutePath testCasesDir) @trusted {
47     DashboardCss.h2(root.addChild(new Link(tag, null)).setAttribute("id", tag[1 .. $]),
48             "Test Cases");
49     auto sections = conf.reportSection.toSet;
50 
51     ReportData data;
52 
53     if (ReportSection.tc_similarity in sections)
54         data.similaritiesData = reportTestCaseSimilarityAnalyse(db, 5);
55 
56     data.addSuggestion = ReportSection.tc_suggestion in sections;
57     // 10 is magic number. feels good.
58     data.classifier = makeTestCaseClassifier(db, 10);
59 
60     const tabGroupName = "testcase_class";
61     Element[Classification] tabLink;
62 
63     { // tab links
64         auto tab = root.addChild("div").addClass("tab");
65         foreach (class_; [EnumMembers!Classification]) {
66             auto b = tab.addChild("button").addClass("tablinks")
67                 .addClass("tablinks_" ~ tabGroupName);
68             b.setAttribute("onclick", format!`openTab(event, '%s', '%s')`(class_, tabGroupName));
69             b.appendText(class_.to!string);
70             tabLink[class_] = b;
71         }
72     }
73 
74     Table[Classification] tabContent;
75     foreach (a; [EnumMembers!Classification]) {
76         auto div = root.addChild("div").addClass("tabcontent")
77             .addClass("tabcontent_" ~ tabGroupName).setAttribute("id", a.to!string);
78         if (a == Classification.Redundant)
79             div.addChild("p", format(classDescription[a], data.classifier.threshold));
80         else
81             div.addChild("p", classDescription[a]);
82         tabContent[a] = tmplSortableTable(div, ["Name", "Tests", "Killed"]);
83     }
84 
85     auto newTestCases = spinSql!(() => db.testCaseApi.getNewTestCases).toSet;
86     logger.trace("new test cases ", newTestCases);
87     long[Classification] classCnt;
88     foreach (tcId; spinSql!(() => db.testCaseApi.getDetectedTestCaseIds)) {
89         const name = spinSql!(() => db.testCaseApi.getTestCaseName(tcId));
90 
91         auto reportFname = TestCase(name).testCaseToHtmlLink;
92         auto fout = File(testCasesDir ~ reportFname, "w");
93         TestCaseSummary summary;
94         spinSql!(() {
95             // do all the heavy database interaction in a transaction to
96             // speedup by reduce locking.
97             auto t = db.transaction;
98             makeTestCasePage(db, name, tcId, (data.similaritiesData is null)
99                 ? null : data.similaritiesData.similarities.get(tcId, null),
100                 data, metaData, summary, fout);
101         });
102 
103         auto classification = classify(TestCase(name), summary, data.classifier, metaData);
104         if (classification == Classification.Buggy && tcId in newTestCases) {
105             logger.trace("test case is new and tagged as buggy: ", name);
106             // only report test cases as buggy if all alive mutants have been
107             // tested. it is only then we can be sure that they actually are buggy.
108         } else {
109             classCnt[classification] += 1;
110 
111             auto r = tabContent[classification].appendRow;
112             {
113                 auto td = r.addChild("td");
114                 td.addChild("a", name).href = buildPath(HtmlStyle.testCaseDir, reportFname);
115             }
116             r.addChild("td", summary.score.to!string);
117             r.addChild("td", summary.kills.to!string);
118         }
119     }
120 
121     foreach (a; classCnt.byKeyValue) {
122         tabLink[a.key].appendText(format!" %s"(a.value));
123         if (auto c = a.key in classColor)
124             tabLink[a.key].style = *c;
125     }
126 }
127 
128 private:
129 
130 enum Classification {
131     Unique,
132     Redundant,
133     Buggy,
134     Normal
135 }
136 
137 immutable string[Classification] classDescription;
138 immutable string[Classification] classColor;
139 
140 shared static this() @trusted {
141     classDescription = cast(immutable)[
142         Classification.Unique: "Kills mutants that no other test case do.",
143         Classification.Redundant: "All mutants the test case kill are also killed by %s other test cases. The test case is probably redudant and thus can be removed.",
144         Classification.Buggy: "Zero killed mutants. The test case is probably incorrect. Immediatly inspect the test case.",
145         Classification.Normal: ""
146     ];
147 
148     classColor = cast(immutable)[
149         // light green
150         Classification.Unique: "background-color: #b3ff99",
151         // light orange
152         Classification.Redundant: "background-color: #ffc266",
153         // light red
154         Classification.Buggy: "background-color: #ff9980"
155     ];
156 }
157 
158 Classification classify(TestCase tc, const TestCaseSummary summary,
159         const ref TestCaseClassifier tclass, const ref TestCaseMetadata metaData) {
160     if (summary.kills == 0)
161         return Classification.Buggy;
162     if (summary.score == 1)
163         return Classification.Unique;
164 
165     if (auto v = tc in metaData.redundant) {
166         if (*v)
167             return Classification.Redundant;
168     } else if (summary.score >= tclass.threshold)
169         return Classification.Redundant;
170 
171     return Classification.Normal;
172 }
173 
174 struct ReportData {
175     TestCaseSimilarityAnalyse similaritiesData;
176     TestCaseClassifier classifier;
177 
178     bool addSuggestion;
179 }
180 
181 struct TestCaseSummary {
182     long kills;
183 
184     // min(f) where f is the number of test cases that killed a mutant.
185     // thus if a test case have one unique mutant the score is 1, none then it
186     // is the lowest of all mutant test case kills.
187     long score;
188 }
189 
190 void makeTestCasePage(ref Database db, const string name, const TestCaseId tcId,
191         TestCaseSimilarityAnalyse.Similarity[] similarities,
192         const ReportData rdata, TestCaseMetadata metaData, ref TestCaseSummary summary, ref File out_) @system {
193     import std.path : baseName;
194     import dextool.plugin.mutate.backend.type : TestCase;
195 
196     auto doc = tmplBasicPage.dashboardCss;
197     scope (success)
198         out_.write(doc.toPrettyString);
199 
200     doc.title(format("%s %s", name, Clock.currTime));
201     doc.mainBody.setAttribute("onload", "init()");
202     doc.root.childElements("head")[0].addChild("script").addChild(new RawSource(doc, jsIndex));
203 
204     doc.mainBody.addChild("h1").appendText(name);
205     if (auto v = TestCase(name) in metaData.loc) {
206         auto p = doc.mainBody.addChild("p");
207         p.addChild("a", v.file.baseName).href = v.file;
208         if (v.line.hasValue)
209             p.appendText(" at line " ~ v.line.orElse(0u).to!string);
210     }
211     if (auto v = TestCase(name) in metaData.text)
212         doc.mainBody.addChild(new RawSource(doc, *v));
213 
214     doc.mainBody.addChild("h2").appendText("Killed");
215     addKilledMutants(db, tcId, rdata, summary, doc.mainBody);
216 
217     if (!similarities.empty) {
218         doc.mainBody.addChild("h2").appendText("Similarity");
219         addSimilarity(db, similarities, doc.mainBody);
220     }
221 }
222 
223 void addKilledMutants(ref Database db, const TestCaseId tcId, const ReportData rdata,
224         ref TestCaseSummary summary, Element root) @system {
225     import std.algorithm : min;
226 
227     auto kills = db.testCaseApi.testCaseKilledSrcMutants(tcId);
228     summary.kills = kills.length;
229     summary.score = kills.length == 0 ? 0 : long.max;
230 
231     auto uniqueElem = root.addChild("div");
232 
233     void addPopupHelp(Element e, string header) {
234         switch (header) {
235         case "TestCases":
236             generatePopupHelp(e, "Number of test cases that kill the mutant.");
237             break;
238         case "Priority":
239             generatePopupHelp(e,
240                     "How important it is to kill the mutant. It is based on modified source code size.");
241             break;
242         case "Suggestion":
243             generatePopupHelp(e, "Alive mutants on the same source code location. Because they are close to a mutant that this test case killed it may be suitable to extend this test case to also kill the suggested mutant.");
244             break;
245         default:
246             break;
247         }
248     }
249 
250     auto tbl = tmplSortableTable(root, ["Link", "TestCases"] ~ (rdata.addSuggestion
251             ? ["Suggestion"] : null) ~ ["Priority", "ExitCode", "Tested"], &addPopupHelp);
252 
253     foreach (const id; kills.sort) {
254         auto r = tbl.appendRow();
255 
256         const info = db.mutantApi.getMutantInfo(id).orElse(MutantInfo2.init);
257 
258         r.addChild("td").addChild("a", format("%s:%s", info.file,
259                 info.sloc.line)).href = format("%s#%s", buildPath("..",
260                 HtmlStyle.fileDir, pathToHtmlLink(info.file)), info.id.get);
261 
262         summary.score = min(info.tcKilled, summary.score);
263         {
264             auto td = r.addChild("td", info.tcKilled.to!string);
265             if (info.tcKilled == 1) {
266                 td.style = "background-color: lightgreen";
267             }
268         }
269 
270         if (rdata.addSuggestion) {
271             auto tds = r.addChild("td");
272             foreach (s; db.mutantApi.getSurroundingAliveMutants(id).enumerate) {
273                 // column sort in the html report do not work correctly if starting from 0.
274                 auto td = tds.addChild("a", format("%s", s.index + 1));
275                 td.href = format("%s#%s", buildPath("..", HtmlStyle.fileDir,
276                         pathToHtmlLink(info.file)), s.value.get);
277                 td.appendText(" ");
278             }
279         }
280         r.addChild("td", info.prio.get.to!string);
281         r.addChild("td", toString(info.exitStatus));
282         r.addChild("td", info.updated.toShortDate);
283     }
284 }
285 
286 void addSimilarity(ref Database db, TestCaseSimilarityAnalyse.Similarity[] similarities,
287         Element root) @system {
288     import dextool.cachetools;
289 
290     auto getPath = nullableCache!(MutationStatusId, string, (MutationStatusId id) {
291         auto path = spinSql!(() => db.mutantApi.getPath(id)).get;
292         return format!"%s#%s"(buildPath("..", HtmlStyle.fileDir, pathToHtmlLink(path)), id.get);
293     })(0, 30.dur!"seconds");
294 
295     root.addChild("p", "How similary this test case is to others.");
296     {
297         auto p = root.addChild("p");
298         p.addChild("b", "Note");
299         p.appendText(": The analysis is based on the mutants that the test cases kill; thus, it is dependent on the mutation operators that are used when generating the report.");
300 
301         root.addChild("p", "The intersection column is the mutants that are killed by both the current test case and in the column Test Case.")
302             .appendText(
303                     " The difference column are the mutants that are only killed by the current test case.");
304     }
305 
306     auto tbl = tmplDefaultTable(root, [
307             "Test Case", "Similarity", "Difference", "Intersection"
308             ]);
309     foreach (const sim; similarities) {
310         auto r = tbl.appendRow();
311 
312         const name = db.testCaseApi.getTestCaseName(sim.testCase);
313         r.addChild("td").addChild("a", name).href = buildPath(TestCase(name).testCaseToHtmlLink);
314 
315         r.addChild("td", format("%#.3s", sim.similarity));
316 
317         auto difference = r.addChild("td");
318         foreach (const mut; sim.difference) {
319             auto link = difference.addChild("a", mut.to!string);
320             link.href = getPath(mut).get;
321             difference.appendText(" ");
322         }
323 
324         auto s = r.addChild("td");
325         foreach (const mut; sim.intersection) {
326             auto link = s.addChild("a", mut.to!string);
327             link.href = getPath(mut).get;
328             s.appendText(" ");
329         }
330     }
331 }