1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.report.html.page_test_case;
11 
12 import logger = std.experimental.logger;
13 import std.algorithm : sort;
14 import std.array : empty;
15 import std.conv : to;
16 import std.datetime : Clock, dur, SysTime;
17 import std.format : format;
18 import std.path : buildPath;
19 import std.range : enumerate;
20 import std.stdio : File;
21 import std.traits : EnumMembers;
22 
23 import arsd.dom : Element, RawSource, Link, Document, Table;
24 import my.optional;
25 import my.path : AbsolutePath;
26 import my.set;
27 
28 import dextool.plugin.mutate.backend.database : Database, spinSql, MutationId,
29     TestCaseId, MutationStatusId, MutantInfo2;
30 import dextool.plugin.mutate.backend.report.analyzers : reportTestCaseUniqueness,
31     TestCaseUniqueness, reportTestCaseSimilarityAnalyse,
32     TestCaseSimilarityAnalyse, TestCaseClassifier, makeTestCaseClassifier, TestCaseMetadata;
33 import dextool.plugin.mutate.backend.report.html.constants : HtmlStyle = Html, DashboardCss;
34 import dextool.plugin.mutate.backend.report.html.tmpl : tmplBasicPage,
35     dashboardCss, tmplSortableTable, tmplDefaultTable;
36 import dextool.plugin.mutate.backend.report.html.utility : pathToHtmlLink, toShortDate;
37 import dextool.plugin.mutate.backend.resource;
38 import dextool.plugin.mutate.backend.type : Mutation, toString, TestCase;
39 import dextool.plugin.mutate.config : ConfigReport;
40 import dextool.plugin.mutate.type : MutationKind, ReportSection;
41 
42 @safe:
43 
44 void makeTestCases(ref Database db, string tag, Element root, ref const ConfigReport conf,
45         const(Mutation.Kind)[] kinds, TestCaseMetadata metaData, AbsolutePath testCasesDir) @trusted {
46     DashboardCss.h2(root.addChild(new Link(tag, null)).setAttribute("id", tag[1 .. $]),
47             "Test Cases");
48     auto sections = conf.reportSection.toSet;
49 
50     ReportData data;
51 
52     if (ReportSection.tc_similarity in sections)
53         data.similaritiesData = reportTestCaseSimilarityAnalyse(db, kinds, 5);
54 
55     data.addSuggestion = ReportSection.tc_suggestion in sections;
56     // 10 is magic number. feels good.
57     data.classifier = makeTestCaseClassifier(db, 10);
58 
59     const tabGroupName = "testcase_class";
60     Element[Classification] tabLink;
61 
62     { // tab links
63         auto tab = root.addChild("div").addClass("tab");
64         foreach (class_; [EnumMembers!Classification]) {
65             auto b = tab.addChild("button").addClass("tablinks")
66                 .addClass("tablinks_" ~ tabGroupName);
67             b.setAttribute("onclick", format!`openTab(event, '%s', '%s')`(class_, tabGroupName));
68             b.appendText(class_.to!string);
69             tabLink[class_] = b;
70         }
71     }
72 
73     Table[Classification] tabContent;
74     foreach (a; [EnumMembers!Classification]) {
75         auto div = root.addChild("div").addClass("tabcontent")
76             .addClass("tabcontent_" ~ tabGroupName).setAttribute("id", a.to!string);
77         if (a == Classification.Redundant)
78             div.addChild("p", format(classDescription[a], data.classifier.threshold));
79         else
80             div.addChild("p", classDescription[a]);
81         tabContent[a] = tmplSortableTable(div, ["Name", "Tests", "Killed"]);
82     }
83 
84     auto newTestCases = spinSql!(() => db.testCaseApi.getNewTestCases).toSet;
85     logger.trace("new test cases ", newTestCases);
86     long[Classification] classCnt;
87     foreach (tcId; spinSql!(() => db.testCaseApi.getDetectedTestCaseIds)) {
88         const name = spinSql!(() => db.testCaseApi.getTestCaseName(tcId));
89 
90         auto reportFname = name.pathToHtmlLink;
91         auto fout = File(testCasesDir ~ reportFname, "w");
92         TestCaseSummary summary;
93         spinSql!(() {
94             // do all the heavy database interaction in a transaction to
95             // speedup by reduce locking.
96             auto t = db.transaction;
97             makeTestCasePage(db, kinds, name, tcId, (data.similaritiesData is null)
98                 ? null : data.similaritiesData.similarities.get(tcId, null),
99                 data, metaData, summary, fout);
100         });
101 
102         auto classification = classify(TestCase(name), summary, data.classifier, metaData);
103         if (classification == Classification.Buggy && tcId in newTestCases) {
104             logger.trace("test case is new and tagged as buggy: ", name);
105             // only report test cases as buggy if all alive mutants have been
106             // tested. it is only then we can be sure that they actually are buggy.
107         } else {
108             classCnt[classification] += 1;
109 
110             auto r = tabContent[classification].appendRow;
111             {
112                 auto td = r.addChild("td");
113                 td.addChild("a", name).href = buildPath(HtmlStyle.testCaseDir, reportFname);
114             }
115             r.addChild("td", summary.score.to!string);
116             r.addChild("td", summary.kills.to!string);
117         }
118     }
119 
120     foreach (a; classCnt.byKeyValue) {
121         tabLink[a.key].appendText(format!" %s"(a.value));
122         if (auto c = a.key in classColor)
123             tabLink[a.key].style = *c;
124     }
125 }
126 
127 private:
128 
129 enum Classification {
130     Unique,
131     Redundant,
132     Buggy,
133     Normal
134 }
135 
136 immutable string[Classification] classDescription;
137 immutable string[Classification] classColor;
138 
139 shared static this() @trusted {
140     classDescription = cast(immutable)[
141         Classification.Unique: "kills mutants that no other test case do.",
142         Classification.Redundant: "all mutants the test case kill are also killed by %s other test cases. The test case is probably redudant and thus can be removed.",
143         Classification.Buggy
144         : "zero killed mutants. The test case is most probably incorrect. Immediatly inspect the test case.",
145         Classification.Normal: ""
146     ];
147 
148     classColor = cast(immutable)[
149         // light green
150         Classification.Unique: "background-color: #b3ff99",
151         // light orange
152         Classification.Redundant: "background-color: #ffc266",
153         // light red
154         Classification.Buggy: "background-color: #ff9980"
155     ];
156 }
157 
158 Classification classify(TestCase tc, const TestCaseSummary summary,
159         const ref TestCaseClassifier tclass, const ref TestCaseMetadata metaData) {
160     if (summary.kills == 0)
161         return Classification.Buggy;
162     if (summary.score == 1)
163         return Classification.Unique;
164 
165     if (auto v = tc in metaData.redundant) {
166         if (*v)
167             return Classification.Redundant;
168     } else if (summary.score >= tclass.threshold)
169         return Classification.Redundant;
170 
171     return Classification.Normal;
172 }
173 
174 struct ReportData {
175     TestCaseSimilarityAnalyse similaritiesData;
176     TestCaseClassifier classifier;
177 
178     bool addSuggestion;
179 }
180 
181 struct TestCaseSummary {
182     long kills;
183 
184     // min(f) where f is the number of test cases that killed a mutant.
185     // thus if a test case have one unique mutant the score is 1, none then it
186     // is the lowest of all mutant test case kills.
187     long score;
188 }
189 
190 void makeTestCasePage(ref Database db, const(Mutation.Kind)[] kinds, const string name,
191         const TestCaseId tcId, TestCaseSimilarityAnalyse.Similarity[] similarities,
192         const ReportData rdata, TestCaseMetadata metaData, ref TestCaseSummary summary, ref File out_) @system {
193     import std.path : baseName;
194     import dextool.plugin.mutate.backend.type : TestCase;
195 
196     auto doc = tmplBasicPage.dashboardCss;
197     scope (success)
198         out_.write(doc.toPrettyString);
199 
200     doc.title(format("%s %s", name, Clock.currTime));
201     doc.mainBody.setAttribute("onload", "init()");
202     doc.root.childElements("head")[0].addChild("script").addChild(new RawSource(doc, jsIndex));
203 
204     doc.mainBody.addChild("h1").appendText(name);
205     if (auto v = TestCase(name) in metaData.loc) {
206         auto p = doc.mainBody.addChild("p");
207         p.addChild("a", v.file.baseName).href = v.file;
208         if (v.line.hasValue)
209             p.appendText(" at line " ~ v.line.orElse(0u).to!string);
210     }
211     if (auto v = TestCase(name) in metaData.text)
212         doc.mainBody.addChild(new RawSource(doc, *v));
213 
214     doc.mainBody.addChild("h2").appendText("Killed");
215     addKilledMutants(db, kinds, tcId, rdata, summary, doc.mainBody);
216 
217     if (!similarities.empty) {
218         doc.mainBody.addChild("h2").appendText("Similarity");
219         addSimilarity(db, similarities, doc.mainBody);
220     }
221 }
222 
223 void addKilledMutants(ref Database db, const(Mutation.Kind)[] kinds,
224         const TestCaseId tcId, const ReportData rdata, ref TestCaseSummary summary, Element root) @system {
225     import std.algorithm : min;
226 
227     auto kills = db.testCaseApi.testCaseKilledSrcMutants(kinds, tcId);
228     summary.kills = kills.length;
229     summary.score = kills.length == 0 ? 0 : long.max;
230 
231     auto uniqueElem = root.addChild("div");
232 
233     {
234         auto p = root.addChild("p");
235         p.addChild("b", "TestCases");
236         p.appendText(": number of test cases that kill the mutant.");
237     }
238     {
239         auto p = root.addChild("p");
240         p.addChild("b", "Suggestion");
241         p.appendText(": alive mutants on the same source code location. Because they are close to a mutant that this test case killed it may be suitable to extend this test case to also kill the suggested mutant.");
242     }
243 
244     auto tbl = tmplSortableTable(root, ["Link", "TestCases"] ~ (rdata.addSuggestion
245             ? ["Suggestion"] : null) ~ ["Priority", "ExitCode", "Tested"]);
246 
247     foreach (const id; kills.sort) {
248         auto r = tbl.appendRow();
249 
250         const info = db.mutantApi.getMutantInfo(id).orElse(MutantInfo2.init);
251 
252         r.addChild("td").addChild("a", format("%s:%s", info.file,
253                 info.sloc.line)).href = format("%s#%s", buildPath("..",
254                 HtmlStyle.fileDir, pathToHtmlLink(info.file)), info.id.get);
255 
256         summary.score = min(info.tcKilled, summary.score);
257         {
258             auto td = r.addChild("td", info.tcKilled.to!string);
259             if (info.tcKilled == 1) {
260                 td.style = "background-color: lightgreen";
261             }
262         }
263 
264         if (rdata.addSuggestion) {
265             auto tds = r.addChild("td");
266             foreach (s; db.mutantApi.getSurroundingAliveMutants(id).enumerate) {
267                 // column sort in the html report do not work correctly if starting from 0.
268                 auto td = tds.addChild("a", format("%s", s.index + 1));
269                 td.href = format("%s#%s", buildPath("..", HtmlStyle.fileDir,
270                         pathToHtmlLink(info.file)), db.mutantApi.getMutationId(s.value).get);
271                 td.appendText(" ");
272             }
273         }
274 
275         r.addChild("td", info.prio.get.to!string);
276         r.addChild("td", toString(info.exitStatus));
277         r.addChild("td", info.updated.toShortDate);
278     }
279 }
280 
281 void addSimilarity(ref Database db, TestCaseSimilarityAnalyse.Similarity[] similarities,
282         Element root) @system {
283     import dextool.cachetools;
284 
285     auto getPath = nullableCache!(MutationStatusId, string, (MutationStatusId id) {
286         auto path = spinSql!(() => db.mutantApi.getPath(id)).get;
287         auto mutId = spinSql!(() => db.mutantApi.getMutationId(id)).get;
288         return format!"%s#%s"(buildPath("..", HtmlStyle.fileDir, pathToHtmlLink(path)), mutId.get);
289     })(0, 30.dur!"seconds");
290 
291     root.addChild("p", "How similary this test case is to others.");
292     {
293         auto p = root.addChild("p");
294         p.addChild("b", "Note");
295         p.appendText(": The analysis is based on the mutants that the test cases kill; thus, it is dependent on the mutation operators that are used when generating the report.");
296 
297         root.addChild("p", "The intersection column is the mutants that are killed by both the current test case and in the column Test Case.")
298             .appendText(
299                     " The difference column are the mutants that are only killed by the current test case.");
300     }
301 
302     auto tbl = tmplDefaultTable(root, [
303             "Test Case", "Similarity", "Difference", "Intersection"
304             ]);
305     foreach (const sim; similarities) {
306         auto r = tbl.appendRow();
307 
308         const name = db.testCaseApi.getTestCaseName(sim.testCase);
309         r.addChild("td").addChild("a", name).href = buildPath(name.pathToHtmlLink);
310 
311         r.addChild("td", format("%#.3s", sim.similarity));
312 
313         auto difference = r.addChild("td");
314         foreach (const mut; sim.difference) {
315             auto link = difference.addChild("a", mut.to!string);
316             link.href = getPath(mut).get;
317             difference.appendText(" ");
318         }
319 
320         auto s = r.addChild("td");
321         foreach (const mut; sim.intersection) {
322             auto link = s.addChild("a", mut.to!string);
323             link.href = getPath(mut).get;
324             s.appendText(" ");
325         }
326     }
327 }