1 /**
2 Copyright: Copyright (c) 2020, Joakim Brännström. All rights reserved.
3 License: MPL-2
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 
6 This Source Code Form is subject to the terms of the Mozilla Public License,
7 v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
8 one at http://mozilla.org/MPL/2.0/.
9 */
10 module dextool.plugin.mutate.backend.test_mutant.schemata;
11 
12 import logger = std.experimental.logger;
13 import std.algorithm : sort, map;
14 import std.array : empty;
15 import std.conv : to;
16 import std.datetime : Duration;
17 import std.exception : collectException;
18 import std.typecons : Tuple;
19 
20 import proc : DrainElement;
21 import sumtype;
22 
23 import dextool.fsm : Fsm, next, act, get, TypeDataMap;
24 
25 import dextool.plugin.mutate.backend.database : MutationStatusId, Database, spinSql;
26 import dextool.plugin.mutate.backend.interface_ : FilesysIO, Blob;
27 import dextool.plugin.mutate.backend.test_mutant.common;
28 import dextool.plugin.mutate.backend.test_mutant.test_cmd_runner;
29 import dextool.plugin.mutate.backend.type : Mutation, TestCase, Checksum;
30 import dextool.plugin.mutate.type : TestCaseAnalyzeBuiltin, ShellCommand;
31 
32 @safe:
33 
34 struct MutationTestResult {
35     import std.datetime : Duration;
36     import dextool.plugin.mutate.backend.database : MutationStatusId;
37     import dextool.plugin.mutate.backend.type : TestCase;
38 
39     MutationStatusId id;
40     Mutation.Status status;
41     Duration testTime;
42     TestCase[] testCases;
43 }
44 
45 struct SchemataTestDriver {
46     private {
47         /// True as long as the schemata driver is running.
48         bool isRunning_ = true;
49 
50         FilesysIO fio;
51 
52         Database* db;
53 
54         /// Runs the test commands.
55         TestRunner* runner;
56 
57         /// Result of testing the mutants.
58         MutationTestResult[] result_;
59     }
60 
61     static struct None {
62     }
63 
64     static struct InitializeData {
65         MutationStatusId[] mutants;
66     }
67 
68     static struct Initialize {
69     }
70 
71     static struct Done {
72     }
73 
74     static struct NextMutantData {
75         /// Mutants to test.
76         InjectIdResult mutants;
77     }
78 
79     static struct NextMutant {
80         bool done;
81         InjectIdResult.InjectId inject;
82     }
83 
84     static struct TestMutantData {
85         /// If the user has configured that the test cases should be analyzed.
86         bool hasTestCaseOutputAnalyzer;
87     }
88 
89     static struct TestMutant {
90         InjectIdResult.InjectId inject;
91 
92         MutationTestResult result;
93         bool hasTestOutput;
94         // if there are mutants status id's related to a file but the mutants
95         // have been removed.
96         bool mutantIdError;
97     }
98 
99     static struct TestCaseAnalyzeData {
100         TestCaseAnalyzer* testCaseAnalyzer;
101         DrainElement[] output;
102     }
103 
104     static struct TestCaseAnalyze {
105         MutationTestResult result;
106         bool unstableTests;
107     }
108 
109     static struct StoreResult {
110         MutationTestResult result;
111     }
112 
113     alias Fsm = dextool.fsm.Fsm!(None, Initialize, Done, NextMutant,
114             TestMutant, TestCaseAnalyze, StoreResult);
115     alias LocalStateDataT = Tuple!(TestMutantData, TestCaseAnalyzeData,
116             NextMutantData, InitializeData);
117 
118     private {
119         Fsm fsm;
120         TypeDataMap!(LocalStateDataT, TestMutant, TestCaseAnalyze, NextMutant, Initialize) local;
121     }
122 
123     this(FilesysIO fio, TestRunner* runner, Database* db,
124             TestCaseAnalyzer* testCaseAnalyzer, MutationStatusId[] mutants) {
125         this.fio = fio;
126         this.runner = runner;
127         this.db = db;
128         this.local.get!Initialize.mutants = mutants;
129         this.local.get!TestCaseAnalyze.testCaseAnalyzer = testCaseAnalyzer;
130         this.local.get!TestMutant.hasTestCaseOutputAnalyzer = !testCaseAnalyzer.empty;
131     }
132 
133     static void execute_(ref SchemataTestDriver self) @trusted {
134         self.fsm.next!((None a) => fsm(Initialize.init),
135                 (Initialize a) => fsm(NextMutant.init), (NextMutant a) {
136             if (a.done)
137                 return fsm(Done.init);
138             return fsm(TestMutant(a.inject));
139         }, (TestMutant a) {
140             if (a.mutantIdError)
141                 return fsm(NextMutant.init);
142             if (a.result.status == Mutation.Status.killed
143                 && self.local.get!TestMutant.hasTestCaseOutputAnalyzer && a.hasTestOutput) {
144                 return fsm(TestCaseAnalyze(a.result));
145             }
146             return fsm(StoreResult(a.result));
147         }, (TestCaseAnalyze a) {
148             if (a.unstableTests)
149                 return fsm(NextMutant.init);
150             return fsm(StoreResult(a.result));
151         }, (StoreResult a) => fsm(NextMutant.init), (Done a) => fsm(a));
152 
153         debug logger.trace("state: ", self.fsm.logNext);
154         self.fsm.act!(self);
155     }
156 
157 nothrow:
158 
159     MutationTestResult[] result() {
160         return result_;
161     }
162 
163     void execute() {
164         try {
165             execute_(this);
166         } catch (Exception e) {
167             logger.warning(e.msg).collectException;
168         }
169     }
170 
171     bool isRunning() {
172         return isRunning_;
173     }
174 
175     void opCall(None data) {
176     }
177 
178     void opCall(Initialize data) {
179         scope (exit)
180             local.get!Initialize.mutants = null;
181 
182         InjectIdBuilder builder;
183         foreach (mutant; local.get!Initialize.mutants) {
184             auto cs = spinSql!(() { return db.getChecksum(mutant); });
185             if (!cs.isNull) {
186                 builder.put(mutant, cs.get);
187             }
188         }
189         debug logger.trace(builder).collectException;
190 
191         local.get!NextMutant.mutants = builder.finalize;
192     }
193 
194     void opCall(Done data) {
195         isRunning_ = false;
196     }
197 
198     void opCall(ref NextMutant data) {
199         data.done = local.get!NextMutant.mutants.empty;
200 
201         if (!data.done) {
202             data.inject = local.get!NextMutant.mutants.front;
203             local.get!NextMutant.mutants.popFront;
204         }
205     }
206 
207     void opCall(ref TestMutant data) {
208         import std.datetime.stopwatch : StopWatch, AutoStart;
209         import dextool.plugin.mutate.backend.analyze.pass_schemata : schemataMutantEnvKey,
210             checksumToId;
211         import dextool.plugin.mutate.backend.generate_mutant : makeMutationText;
212 
213         data.result.id = data.inject.statusId;
214 
215         auto id = spinSql!(() { return db.getMutationId(data.inject.statusId); });
216         if (id.isNull) {
217             data.mutantIdError = true;
218             return;
219         }
220         auto entry_ = spinSql!(() { return db.getMutation(id.get); });
221         if (entry_.isNull) {
222             data.mutantIdError = true;
223             return;
224         }
225         auto entry = entry_.get;
226 
227         try {
228             const file = fio.toAbsoluteRoot(entry.file);
229             auto original = fio.makeInput(file);
230             auto txt = makeMutationText(original, entry.mp.offset,
231                     entry.mp.mutations[0].kind, entry.lang);
232             debug logger.trace(entry);
233             logger.infof("%s from '%s' to '%s' in %s:%s:%s", data.inject.injectId,
234                     txt.original, txt.mutation, file, entry.sloc.line, entry.sloc.column);
235         } catch (Exception e) {
236             logger.info(e.msg).collectException;
237         }
238 
239         runner.env[schemataMutantEnvKey] = data.inject.injectId.to!string;
240         scope (exit)
241             runner.env.remove(schemataMutantEnvKey);
242 
243         auto sw = StopWatch(AutoStart.yes);
244         auto res = runTester(*runner);
245         data.result.testTime = sw.peek;
246 
247         data.result.status = res.status;
248         data.hasTestOutput = !res.output.empty;
249         local.get!TestCaseAnalyze.output = res.output;
250 
251         logger.infof("%s %s (%s)", data.inject.injectId, data.result.status,
252                 data.result.testTime).collectException;
253     }
254 
255     void opCall(ref TestCaseAnalyze data) {
256         try {
257             auto analyze = local.get!TestCaseAnalyze.testCaseAnalyzer.analyze(
258                     local.get!TestCaseAnalyze.output);
259             local.get!TestCaseAnalyze.output = null;
260 
261             analyze.match!((TestCaseAnalyzer.Success a) {
262                 data.result.testCases = a.failed;
263             }, (TestCaseAnalyzer.Unstable a) {
264                 logger.warningf("Unstable test cases found: [%-(%s, %)]", a.unstable);
265                 logger.info(
266                     "As configured the result is ignored which will force the mutant to be re-tested");
267                 data.unstableTests = true;
268             }, (TestCaseAnalyzer.Failed a) {
269                 logger.warning("The parser that analyze the output from test case(s) failed");
270             });
271 
272             logger.infof(!data.result.testCases.empty, `%s killed by [%-(%s, %)]`,
273                     data.result.id, data.result.testCases.sort.map!"a.name").collectException;
274         } catch (Exception e) {
275             logger.warning(e.msg).collectException;
276         }
277     }
278 
279     void opCall(StoreResult data) {
280         result_ ~= data.result;
281     }
282 }
283 
284 /** Generate schemata injection IDs (32bit) from mutant checksums (128bit).
285  *
286  * There is a possibility that an injection ID result in a collision because
287  * they are only 32 bit. If that happens the mutant is discarded as unfeasable
288  * to use for schemata.
289  *
290  * TODO: if this is changed to being order dependent then it can handle all
291  * mutants. But I can't see how that can be done easily both because of how the
292  * schemas are generated and how the database is setup.
293  */
294 struct InjectIdBuilder {
295     import dextool.set;
296 
297     private {
298         alias InjectId = InjectIdResult.InjectId;
299 
300         InjectId[uint] result;
301         Set!uint collisions;
302     }
303 
304     void put(MutationStatusId id, Checksum cs) @safe pure nothrow {
305         import dextool.plugin.mutate.backend.analyze.pass_schemata : checksumToId;
306 
307         const injectId = checksumToId(cs);
308         debug logger.tracef("%s %s %s", id, cs, injectId).collectException;
309 
310         if (injectId in collisions) {
311         } else if (injectId in result) {
312             collisions.add(injectId);
313             result.remove(injectId);
314         } else {
315             result[injectId] = InjectId(id, injectId);
316         }
317     }
318 
319     InjectIdResult finalize() @safe pure nothrow {
320         import std.array : array;
321 
322         return InjectIdResult(result.byValue.array);
323     }
324 }
325 
326 struct InjectIdResult {
327     alias InjectId = Tuple!(MutationStatusId, "statusId", uint, "injectId");
328     InjectId[] ids;
329 
330     InjectId front() @safe pure nothrow {
331         assert(!empty, "Can't get front of an empty range");
332         return ids[0];
333     }
334 
335     void popFront() @safe pure nothrow {
336         assert(!empty, "Can't pop front of an empty range");
337         ids = ids[1 .. $];
338     }
339 
340     bool empty() @safe pure nothrow const @nogc {
341         return ids.empty;
342     }
343 }
344 
345 @("shall detect a collision and make sure it is never part of the result")
346 unittest {
347     InjectIdBuilder builder;
348     builder.put(MutationStatusId(1), Checksum(1, 2));
349     builder.put(MutationStatusId(2), Checksum(3, 4));
350     builder.put(MutationStatusId(3), Checksum(1, 2));
351     auto r = builder.finalize;
352 
353     assert(r.front.statusId == MutationStatusId(2));
354     r.popFront;
355     assert(r.empty);
356 }