1 /** 2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved. 3 License: MPL-2 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 6 This Source Code Form is subject to the terms of the Mozilla Public License, 7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain 8 one at http://mozilla.org/MPL/2.0/. 9 */ 10 module dextool.plugin.mutate.backend.report.html.page_test_case; 11 12 import logger = std.experimental.logger; 13 import std.algorithm : sort; 14 import std.array : empty; 15 import std.conv : to; 16 import std.datetime : Clock, dur, SysTime; 17 import std.format : format; 18 import std.path : buildPath; 19 import std.range : enumerate; 20 import std.stdio : File; 21 import std.traits : EnumMembers; 22 23 import arsd.dom : Element, RawSource, Link, Document, Table; 24 import my.optional; 25 import my.path : AbsolutePath; 26 import my.set; 27 28 import dextool.plugin.mutate.backend.database : Database, spinSql, MutationId, 29 TestCaseId, MutationStatusId, MutantInfo2; 30 import dextool.plugin.mutate.backend.report.analyzers : reportTestCaseUniqueness, 31 TestCaseUniqueness, reportTestCaseSimilarityAnalyse, 32 TestCaseSimilarityAnalyse, TestCaseClassifier, makeTestCaseClassifier, TestCaseMetadata; 33 import dextool.plugin.mutate.backend.report.html.constants : HtmlStyle = Html, DashboardCss; 34 import dextool.plugin.mutate.backend.report.html.tmpl : tmplBasicPage, 35 dashboardCss, tmplSortableTable, tmplDefaultTable; 36 import dextool.plugin.mutate.backend.report.html.utility : pathToHtmlLink, toShortDate; 37 import dextool.plugin.mutate.backend.resource; 38 import dextool.plugin.mutate.backend.type : Mutation, toString, TestCase; 39 import dextool.plugin.mutate.config : ConfigReport; 40 import dextool.plugin.mutate.type : MutationKind, ReportSection; 41 42 @safe: 43 44 void makeTestCases(ref Database db, string tag, Element root, ref const ConfigReport conf, 45 const(Mutation.Kind)[] kinds, TestCaseMetadata metaData, AbsolutePath testCasesDir) @trusted { 46 DashboardCss.h2(root.addChild(new Link(tag, null)).setAttribute("id", tag[1 .. $]), 47 "Test Cases"); 48 auto sections = conf.reportSection.toSet; 49 50 ReportData data; 51 52 if (ReportSection.tc_similarity in sections) 53 data.similaritiesData = reportTestCaseSimilarityAnalyse(db, kinds, 5); 54 55 data.addSuggestion = ReportSection.tc_suggestion in sections; 56 // 10 is magic number. feels good. 57 data.classifier = makeTestCaseClassifier(db, 10); 58 59 const tabGroupName = "testcase_class"; 60 Element[Classification] tabLink; 61 62 { // tab links 63 auto tab = root.addChild("div").addClass("tab"); 64 foreach (class_; [EnumMembers!Classification]) { 65 auto b = tab.addChild("button").addClass("tablinks") 66 .addClass("tablinks_" ~ tabGroupName); 67 b.setAttribute("onclick", format!`openTab(event, '%s', '%s')`(class_, tabGroupName)); 68 b.appendText(class_.to!string); 69 tabLink[class_] = b; 70 } 71 } 72 73 Table[Classification] tabContent; 74 foreach (a; [EnumMembers!Classification]) { 75 auto div = root.addChild("div").addClass("tabcontent") 76 .addClass("tabcontent_" ~ tabGroupName).setAttribute("id", a.to!string); 77 if (a == Classification.Redundant) 78 div.addChild("p", format(classDescription[a], data.classifier.threshold)); 79 else 80 div.addChild("p", classDescription[a]); 81 tabContent[a] = tmplSortableTable(div, ["Name", "Tests", "Killed"]); 82 } 83 84 auto newTestCases = spinSql!(() => db.testCaseApi.getNewTestCases).toSet; 85 logger.trace("new test cases ", newTestCases); 86 long[Classification] classCnt; 87 foreach (tcId; spinSql!(() => db.testCaseApi.getDetectedTestCaseIds)) { 88 const name = spinSql!(() => db.testCaseApi.getTestCaseName(tcId)); 89 90 auto reportFname = name.pathToHtmlLink; 91 auto fout = File(testCasesDir ~ reportFname, "w"); 92 TestCaseSummary summary; 93 spinSql!(() { 94 // do all the heavy database interaction in a transaction to 95 // speedup by reduce locking. 96 auto t = db.transaction; 97 makeTestCasePage(db, kinds, name, tcId, (data.similaritiesData is null) 98 ? null : data.similaritiesData.similarities.get(tcId, null), 99 data, metaData, summary, fout); 100 }); 101 102 auto classification = classify(TestCase(name), summary, data.classifier, metaData); 103 if (classification == Classification.Buggy && tcId in newTestCases) { 104 logger.trace("test case is new and tagged as buggy: ", name); 105 // only report test cases as buggy if all alive mutants have been 106 // tested. it is only then we can be sure that they actually are buggy. 107 } else { 108 classCnt[classification] += 1; 109 110 auto r = tabContent[classification].appendRow; 111 { 112 auto td = r.addChild("td"); 113 td.addChild("a", name).href = buildPath(HtmlStyle.testCaseDir, reportFname); 114 } 115 r.addChild("td", summary.score.to!string); 116 r.addChild("td", summary.kills.to!string); 117 } 118 } 119 120 foreach (a; classCnt.byKeyValue) { 121 tabLink[a.key].appendText(format!" %s"(a.value)); 122 if (auto c = a.key in classColor) 123 tabLink[a.key].style = *c; 124 } 125 } 126 127 private: 128 129 enum Classification { 130 Unique, 131 Redundant, 132 Buggy, 133 Normal 134 } 135 136 immutable string[Classification] classDescription; 137 immutable string[Classification] classColor; 138 139 shared static this() @trusted { 140 classDescription = cast(immutable)[ 141 Classification.Unique: "kills mutants that no other test case do.", 142 Classification.Redundant: "all mutants the test case kill are also killed by %s other test cases. The test case is probably redudant and thus can be removed.", 143 Classification.Buggy 144 : "zero killed mutants. The test case is most probably incorrect. Immediatly inspect the test case.", 145 Classification.Normal: "" 146 ]; 147 148 classColor = cast(immutable)[ 149 // light green 150 Classification.Unique: "background-color: #b3ff99", 151 // light orange 152 Classification.Redundant: "background-color: #ffc266", 153 // light red 154 Classification.Buggy: "background-color: #ff9980" 155 ]; 156 } 157 158 Classification classify(TestCase tc, const TestCaseSummary summary, 159 const ref TestCaseClassifier tclass, const ref TestCaseMetadata metaData) { 160 if (summary.kills == 0) 161 return Classification.Buggy; 162 if (summary.score == 1) 163 return Classification.Unique; 164 165 if (auto v = tc in metaData.redundant) { 166 if (*v) 167 return Classification.Redundant; 168 } else if (summary.score >= tclass.threshold) 169 return Classification.Redundant; 170 171 return Classification.Normal; 172 } 173 174 struct ReportData { 175 TestCaseSimilarityAnalyse similaritiesData; 176 TestCaseClassifier classifier; 177 178 bool addSuggestion; 179 } 180 181 struct TestCaseSummary { 182 long kills; 183 184 // min(f) where f is the number of test cases that killed a mutant. 185 // thus if a test case have one unique mutant the score is 1, none then it 186 // is the lowest of all mutant test case kills. 187 long score; 188 } 189 190 void makeTestCasePage(ref Database db, const(Mutation.Kind)[] kinds, const string name, 191 const TestCaseId tcId, TestCaseSimilarityAnalyse.Similarity[] similarities, 192 const ReportData rdata, TestCaseMetadata metaData, ref TestCaseSummary summary, ref File out_) @system { 193 import std.path : baseName; 194 import dextool.plugin.mutate.backend.type : TestCase; 195 196 auto doc = tmplBasicPage.dashboardCss; 197 scope (success) 198 out_.write(doc.toPrettyString); 199 200 doc.title(format("%s %s", name, Clock.currTime)); 201 doc.mainBody.setAttribute("onload", "init()"); 202 doc.root.childElements("head")[0].addChild("script").addChild(new RawSource(doc, jsIndex)); 203 204 doc.mainBody.addChild("h1").appendText(name); 205 if (auto v = TestCase(name) in metaData.loc) { 206 auto p = doc.mainBody.addChild("p"); 207 p.addChild("a", v.file.baseName).href = v.file; 208 if (v.line.hasValue) 209 p.appendText(" at line " ~ v.line.orElse(0u).to!string); 210 } 211 if (auto v = TestCase(name) in metaData.text) 212 doc.mainBody.addChild(new RawSource(doc, *v)); 213 214 doc.mainBody.addChild("h2").appendText("Killed"); 215 addKilledMutants(db, kinds, tcId, rdata, summary, doc.mainBody); 216 217 if (!similarities.empty) { 218 doc.mainBody.addChild("h2").appendText("Similarity"); 219 addSimilarity(db, similarities, doc.mainBody); 220 } 221 } 222 223 void addKilledMutants(ref Database db, const(Mutation.Kind)[] kinds, 224 const TestCaseId tcId, const ReportData rdata, ref TestCaseSummary summary, Element root) @system { 225 import std.algorithm : min; 226 227 auto kills = db.testCaseApi.testCaseKilledSrcMutants(kinds, tcId); 228 summary.kills = kills.length; 229 summary.score = kills.length == 0 ? 0 : long.max; 230 231 auto uniqueElem = root.addChild("div"); 232 233 { 234 auto p = root.addChild("p"); 235 p.addChild("b", "TestCases"); 236 p.appendText(": number of test cases that kill the mutant."); 237 } 238 { 239 auto p = root.addChild("p"); 240 p.addChild("b", "Suggestion"); 241 p.appendText(": alive mutants on the same source code location. Because they are close to a mutant that this test case killed it may be suitable to extend this test case to also kill the suggested mutant."); 242 } 243 244 auto tbl = tmplSortableTable(root, ["Link", "TestCases"] ~ (rdata.addSuggestion 245 ? ["Suggestion"] : null) ~ ["Priority", "ExitCode", "Tested"]); 246 247 foreach (const id; kills.sort) { 248 auto r = tbl.appendRow(); 249 250 const info = db.mutantApi.getMutantInfo(id).orElse(MutantInfo2.init); 251 252 r.addChild("td").addChild("a", format("%s:%s", info.file, 253 info.sloc.line)).href = format("%s#%s", buildPath("..", 254 HtmlStyle.fileDir, pathToHtmlLink(info.file)), info.id.get); 255 256 summary.score = min(info.tcKilled, summary.score); 257 { 258 auto td = r.addChild("td", info.tcKilled.to!string); 259 if (info.tcKilled == 1) { 260 td.style = "background-color: lightgreen"; 261 } 262 } 263 264 if (rdata.addSuggestion) { 265 auto tds = r.addChild("td"); 266 foreach (s; db.mutantApi.getSurroundingAliveMutants(id).enumerate) { 267 // column sort in the html report do not work correctly if starting from 0. 268 auto td = tds.addChild("a", format("%s", s.index + 1)); 269 td.href = format("%s#%s", buildPath("..", HtmlStyle.fileDir, 270 pathToHtmlLink(info.file)), db.mutantApi.getMutationId(s.value).get); 271 td.appendText(" "); 272 } 273 } 274 275 r.addChild("td", info.prio.get.to!string); 276 r.addChild("td", toString(info.exitStatus)); 277 r.addChild("td", info.updated.toShortDate); 278 } 279 } 280 281 void addSimilarity(ref Database db, TestCaseSimilarityAnalyse.Similarity[] similarities, 282 Element root) @system { 283 import dextool.cachetools; 284 285 auto getPath = nullableCache!(MutationStatusId, string, (MutationStatusId id) { 286 auto path = spinSql!(() => db.mutantApi.getPath(id)).get; 287 auto mutId = spinSql!(() => db.mutantApi.getMutationId(id)).get; 288 return format!"%s#%s"(buildPath("..", HtmlStyle.fileDir, pathToHtmlLink(path)), mutId.get); 289 })(0, 30.dur!"seconds"); 290 291 root.addChild("p", "How similary this test case is to others."); 292 { 293 auto p = root.addChild("p"); 294 p.addChild("b", "Note"); 295 p.appendText(": The analysis is based on the mutants that the test cases kill; thus, it is dependent on the mutation operators that are used when generating the report."); 296 297 root.addChild("p", "The intersection column is the mutants that are killed by both the current test case and in the column Test Case.") 298 .appendText( 299 " The difference column are the mutants that are only killed by the current test case."); 300 } 301 302 auto tbl = tmplDefaultTable(root, [ 303 "Test Case", "Similarity", "Difference", "Intersection" 304 ]); 305 foreach (const sim; similarities) { 306 auto r = tbl.appendRow(); 307 308 const name = db.testCaseApi.getTestCaseName(sim.testCase); 309 r.addChild("td").addChild("a", name).href = buildPath(name.pathToHtmlLink); 310 311 r.addChild("td", format("%#.3s", sim.similarity)); 312 313 auto difference = r.addChild("td"); 314 foreach (const mut; sim.difference) { 315 auto link = difference.addChild("a", mut.to!string); 316 link.href = getPath(mut).get; 317 difference.appendText(" "); 318 } 319 320 auto s = r.addChild("td"); 321 foreach (const mut; sim.intersection) { 322 auto link = s.addChild("a", mut.to!string); 323 link.href = getPath(mut).get; 324 s.appendText(" "); 325 } 326 } 327 }